Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Recorded before the first save, so a crash right after minting
773        // cannot leave a run with no origin. Legibility only: nothing reads it
774        // to decide anything.
775        state.fetched_base = Some(fetched_base);
776        state.origin = Some(origin);
777        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778        state.status = RunStatus::Prep;
779        state.event("start", format!("run {} created", state.id));
780        startup.finish(&mut state, Ok(()))?;
781        Ok(Self {
782            state,
783            roles,
784            sem: Arc::new(Semaphore::new(max_parallel)),
785            pause: Pause::new(),
786            interrupt: Pause::new(),
787        })
788    }
789
790    /// Open a review-only run against work that already exists on `branch`.
791    ///
792    /// The expensive half of the graph is the implement wave — measured at
793    /// 111 and 134 internal tool-loop turns on this repository, against a
794    /// handful for a judge or a reviewer. The cheap half is worth running on
795    /// hand-written work too, and there was no way to reach it.
796    ///
797    /// No new state and no schema change are needed: a run with **one** viable
798    /// candidate and a tally already decided degrades `execute` to exactly
799    /// review → gate → merge, because `judge` skips a single-candidate field,
800    /// `deliberate` has fewer than two first choices to reconcile, `vote`
801    /// returns early, `tally` is already present and `fold_losers` has no
802    /// losers. Resuming such a run therefore does the right thing as well.
803    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804        Self::review_taking_over(repo, branch, config, None, origin).await
805    }
806
807    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
808    /// at the same task still has `branch` checked out, its worktree is
809    /// released first if that is safe (see [`crate::handover`]), and the
810    /// review refuses with the reason if it is not. `None` is a hand-run
811    /// review: it has no earlier attempts, so only a worktree of a dead run
812    /// magi recorded itself can be released.
813    pub async fn review_taking_over(
814        repo: &Path,
815        branch: &str,
816        config: Config,
817        takeover: Option<crate::handover::Takeover>,
818        origin: Origin,
819    ) -> Result<Self> {
820        let repo = git::toplevel(repo).await?;
821        let missing = agent::missing_programs(&config.agents);
822        if !missing.is_empty() {
823            bail!(
824                "these agent programs are not on PATH: {}. Fix the roster in \
825                 magi.toml or install them.",
826                missing.join(", ")
827            );
828        }
829        let base_branch =
830            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831                .await?;
832        if base_branch == branch {
833            bail!("`{branch}` is the base branch; there is nothing to review against");
834        }
835        let fetched_base = resolve_base(
836            &repo,
837            &base_branch,
838            &config.merge.remote,
839            config.coordination.as_ref(),
840        )
841        .await?;
842        let base_commit = fetched_base.commit.clone();
843
844        let roles = config.resolve_roles()?;
845        let max_parallel = config.graph.max_parallel.max(1);
846        let mut state = RunState::new(
847            repo.clone(),
848            base_branch,
849            base_commit.clone(),
850            format!("Review the work already on branch `{branch}`."),
851            config,
852        );
853        state.fetched_base = Some(fetched_base);
854        state.origin = Some(origin);
855        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857        let opened = async {
858            startup.stage(&mut state, "handover")?;
859
860            // Released before anything else touches the branch: a stale local
861            // branch is moved with `git branch -f`, which git refuses while an
862            // earlier attempt's worktree still has it checked out. Everything
863            // after this point that can fail puts the old run back.
864            // A hand-run review has no task, hence no earlier attempts, but a
865            // worktree of a dead run magi made may still be released.
866            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867                earlier: Vec::new(),
868                home: crate::run::home(),
869                choice: None,
870                task_instruction: None,
871            });
872            let wait = repository_wait(&state, None);
873            let coordination_config = state.config.coordination.clone();
874            let coordination =
875                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876                    .await?;
877            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878            if let Some(released) = &released {
879                state.event(
880                    "release",
881                    format!(
882                        "took `{branch}` over from run {}: its worktree was released: {}",
883                        crate::run::short_of(&released.old_id),
884                        released.audit
885                    ),
886                );
887            }
888            let opened = async {
889                startup.stage(&mut state, "review-preflight")?;
890                // The owner's answer to an earlier divergence question is applied
891                // here: after the release (git will not move a checked-out branch)
892                // and before the sync that would otherwise ask again.
893                state.foreign_checkout = crate::handover::preflight_review(
894                    &repo,
895                    branch,
896                    &takeover.home,
897                    &state.config.merge.remote,
898                )
899                .await?;
900                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902                }
903                if let Some(choice) = takeover.choice.as_ref()
904                    && let Err(e) =
905                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906                            .await
907                {
908                    return Err(e.context("applying the owner's answer about the diverged branch"));
909                }
910                // An earlier attempt of this task that published the task head, and
911                // whose work is the branch under review, hands that head over: the
912                // review run updates the same pull request. Without one (a legacy
913                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
914                // own head and its own open pull request is adopted as before.
915                for id in takeover.earlier.iter().rev() {
916                    if let Ok(old) = RunState::load_under(id, &takeover.home)
917                        && let Some(head) = old.pr_head.clone()
918                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919                    {
920                        state.pr_head = Some(head);
921                        break;
922                    }
923                }
924                drop(coordination);
925                startup.stage(&mut state, "open-review")?;
926                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927            }
928            .await;
929            if opened.is_err()
930                && let Some(released) = &released
931            {
932                // Restoration is ownership checked and best effort. Keep the original
933                // startup cause even if reacquiring coordination is no longer possible.
934                match crate::repo_lock::SeatLease::operation_with(
935                    &repo,
936                    coordination_config.as_ref(),
937                    &wait,
938                )
939                .await
940                {
941                    Ok(_coordination) => {
942                        released.restore(&repo, branch).await;
943                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
944                            .map(|old| {
945                                format!(
946                                    "predecessor released_to={:?}; candidate path exists={}",
947                                    old.released_to,
948                                    old.candidates.iter().any(|c| {
949                                        c.branch == branch && c.worktree.exists()
950                                    }),
951                                )
952                            })
953                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955                    }
956                    Err(error) => {
957                        tracing::warn!(
958                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959                            state.id
960                        );
961                        state.event(
962                            "startup-restore",
963                            format!("restoration not attempted: {error:#}"),
964                        );
965                    }
966                }
967            }
968            opened
969        }
970        .await;
971        startup.finish(&mut state, opened)?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Recover exactly the operator-selected public implementation, without touching old sources.
982    pub async fn recover_pr(
983        repo: &Path,
984        task: &str,
985        selection: crate::pr_recovery::Selection,
986        mut config: Config,
987    ) -> Result<Self> {
988        selection.configure(&mut config)?;
989        selection.check_task(task, &selection.instruction)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.origin = Some(crate::run::Origin::queue(task));
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.runs.last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.ensure_base_episode();
1417        state.record_base_sync_stop(&head);
1418        state.save()?;
1419        let fetched =
1420            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421        anyhow::ensure!(
1422            fetched.ok(),
1423            "cannot authorize base recovery without a fetched base"
1424        );
1425        let approved_tip = git::rev_parse(
1426            &state.repo,
1427            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428        )
1429        .await?;
1430        if resumption {
1431            anyhow::ensure!(
1432                crate::daemon::compatible_run_instruction(&state, &queued),
1433                "resumption instruction does not match the retained run"
1434            );
1435            let approval = queued
1436                .resumption
1437                .as_ref()
1438                .context("no operator resumption")?;
1439            anyhow::ensure!(
1440                approval.instruction == queued.instruction,
1441                "resumption approval does not match the current task instruction"
1442            );
1443            state.authorize_base_sync_resumption(task, approval)?;
1444        } else {
1445            state.authorize_base_sync(task)?;
1446        }
1447        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448        state
1449            .base_sync_recovery
1450            .authorizations
1451            .last_mut()
1452            .unwrap()
1453            .scope = Some(crate::base_sync_recovery::Scope {
1454            tip: approved_tip,
1455            instruction: queued.instruction.clone(),
1456            episode,
1457        });
1458        state.event(
1459            "base-recovery",
1460            "owner authorized a bounded base synchronization interval; fresh verification required",
1461        );
1462        state.save()?;
1463        Ok(())
1464    }
1465
1466    /// Reopen an existing run.
1467    pub fn resume(id: &str) -> Result<Self> {
1468        let state = RunState::load(id)?;
1469        if state.startup.as_ref().is_some_and(|s| !s.complete)
1470            || (state.status == RunStatus::Prep
1471                && state.instruction.trim().is_empty()
1472                && state.candidates.is_empty()
1473                && state.prep_complete == Some(false))
1474        {
1475            bail!(
1476                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477            );
1478        }
1479        crate::pr_head::ensure_owner(&state)?;
1480        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481            bail!(
1482                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483            );
1484        }
1485        if let Some(to) = &state.released_to {
1486            bail!(
1487                "run {} cannot be resumed: its worktree was released to run {}",
1488                state.short(),
1489                crate::run::short_of(to)
1490            );
1491        }
1492        let roles = state.config.resolve_roles()?;
1493        if let Some(id) = &state.config.graph.selected_implementer
1494            && state.candidates.iter().any(|c| c.agent != *id)
1495        {
1496            bail!(
1497                "replacement implementer pin does not match the saved candidates; task must stay held"
1498            );
1499        }
1500        let max_parallel = state.config.graph.max_parallel.max(1);
1501        Ok(Self {
1502            state,
1503            roles,
1504            sem: Arc::new(Semaphore::new(max_parallel)),
1505            pause: Pause::new(),
1506            interrupt: Pause::new(),
1507        })
1508    }
1509
1510    /// Walk the graph to a terminal state, skipping nodes already recorded.
1511    ///
1512    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1513    /// the phone - ends here, so this is the one place a run that ended
1514    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1515    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1516    pub async fn execute(&mut self) -> Result<()> {
1517        anyhow::ensure!(
1518            self.state.startup.as_ref().is_none_or(|s| s.complete),
1519            "startup incomplete; inspect retained evidence before explicit recovery"
1520        );
1521        if let Some(selection) = &self.state.pr_recovery
1522            && (selection.source_proof.is_some() || selection.source_run.is_some())
1523        {
1524            Box::pin(selection.verify_source(&self.state.repo)).await?;
1525        }
1526        // Queue claims cover daemon work; this also serializes manual drivers
1527        // consuming a repair answer, including the new review and decision.
1528        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529            || self.state.land_approval.is_some()
1530            || self.state.merge_decision.is_some()
1531            || !self.state.merge_repairs.is_empty()
1532            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533        {
1534            let claim = FixClaim::acquire(&self.state.dir())?;
1535            let disk = RunState::load(&self.state.id)?;
1536            let checkpoint = |s: &RunState| {
1537                (
1538                    s.merge_repairs.len(),
1539                    s.merge_repairs
1540                        .last()
1541                        .map(|r| (r.question.clone(), r.phase)),
1542                    s.land_approval
1543                        .as_ref()
1544                        .map(|a| (a.question.clone(), a.head.clone())),
1545                )
1546            };
1547            if checkpoint(&disk) != checkpoint(&self.state) {
1548                bail!(
1549                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550                );
1551            }
1552            Some(claim)
1553        } else {
1554            None
1555        };
1556        let mut execution = self.claim_driver().await?;
1557        let runner = &mut *execution.runner;
1558        let wait = runner.repository_wait();
1559        // Keep the graph's large future off the callers' stack, including
1560        // standalone task attempts running on a default-sized test thread.
1561        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562        let result = match result {
1563            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564                record_repository_block(&mut runner.state, &e);
1565                runner.park_here()?;
1566                Ok(())
1567            }
1568            other => other,
1569        };
1570        execution.driver.finish(&mut runner.state).await?;
1571        let ended = if result.is_err() {
1572            Some({
1573                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574                if let Err(error) = &result {
1575                    notice.message = format!(
1576                        "Run {} stopped with an error: {}",
1577                        runner.state.id,
1578                        error.to_string().lines().next().unwrap_or("unknown error")
1579                    );
1580                }
1581                notice
1582            })
1583        } else {
1584            crate::notices::run_ended(&runner.state)
1585        };
1586        if let Some(notice) = ended {
1587            crate::notices::raise(notice);
1588        }
1589        result
1590    }
1591
1592    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594    }
1595
1596    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597        let wait = self.repository_wait();
1598        coordinate_operation(&mut self.state, cwd, &wait).await
1599    }
1600
1601    /// Validate ownership and persist the driver claim in one critical section.
1602    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603        let owner = match crate::pr_head::owner_lock_wait(
1604            &crate::run::home(),
1605            &self.state.id,
1606            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607        )
1608        .await
1609        {
1610            Ok(owner) => owner,
1611            Err(error) => {
1612                let detail = format!("worktree maintenance or ownership update: {error:#}");
1613                // Admission did not own run.json: retain this resource outcome
1614                // in memory for queue settlement without overwriting a concurrent
1615                // retirement or publication transaction.
1616                self.state.resource_block = Some(crate::run::ResourceBlock {
1617                    cause: crate::run::ResourceBlockCause::Other,
1618                    node: "start".into(),
1619                    kind: "worktree maintenance".into(),
1620                    detail: detail.clone(),
1621                    at: Timestamp::now(),
1622                });
1623                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624            }
1625        };
1626        let disk = RunState::load(&self.state.id)?;
1627        crate::pr_head::ensure_owner(&disk)?;
1628        let claimed =
1629            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630        // Only proven-live ownership refuses here. Unknown records (legacy runs
1631        // with no claim marker or process identity) fall through to the
1632        // exclusive execution lease taken just below, which is what actually
1633        // keeps a second driver out.
1634        anyhow::ensure!(
1635            disk.liveness(claimed) != crate::run::Liveness::Live,
1636            "run already has a live driver; wait for its execution to end"
1637        );
1638        if let Some(to) = &disk.released_to {
1639            bail!(
1640                "run {} cannot continue: its worktree was released to run {}",
1641                self.state.short(),
1642                crate::run::short_of(to)
1643            );
1644        }
1645        anyhow::ensure!(
1646            serde_json::to_value(&disk.base_sync_recovery)?
1647                == serde_json::to_value(&self.state.base_sync_recovery)?,
1648            "base recovery changed since this run was loaded; reload the saved run"
1649        );
1650        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651        let mut execution = DriverExecution {
1652            runner: self,
1653            driver,
1654        };
1655        // Declare after the execution guard: on any claim error this short
1656        // lock must drop before that guard reloads and releases ownership.
1657        let _owner = owner;
1658        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659            .await?;
1660        let state = &mut execution.runner.state;
1661        // Moving again, so it is no longer parked. Set before the walk rather
1662        // than in `resume`, so every way of re-entering the graph clears it
1663        // and a card cannot claim a run is waiting to be resumed while the
1664        // agents are already working.
1665        state.parked = false;
1666        state.resource_wait = None;
1667        state.resource_block = None;
1668        // Any seat this state still lists as answering belongs to whatever
1669        // process last drove this run — this one included, if it crashed
1670        // mid-wave. Cleared and flushed immediately, before anything else
1671        // runs, so a resume can never show a seat as live when nothing is
1672        // asking it anything yet; the node that actually dispatches the next
1673        // wave repopulates it.
1674        state.clear_active();
1675        // Recorded in the same spot, and flushed together with the clear
1676        // above: this is the pid a reader checks (`RunState::liveness`) when
1677        // no daemon claim exists to answer "is a process still driving this
1678        // run" — a plain `magi run` / `magi review` typed into a terminal
1679        // claims nothing there. Always overwritten, never only-if-absent, so
1680        // a resumed run's stale pid from a previous, possibly-dead process
1681        // can never survive into this one's own report. Unlike
1682        // `clear_active`, this changes on every single `execute()` call, so
1683        // the save below is now unconditional rather than only-if-cleared.
1684        //
1685        // `driver_started_at` is recorded in the same breath, from this same
1686        // pid, so `liveness` can tell a live pid that is genuinely still us
1687        // apart from one the OS has since handed to an unrelated process —
1688        // see that field's own doc for why the pid alone is not enough.
1689        let pid = std::process::id();
1690        state.driver_pid = Some(pid);
1691        state.driver_started_at = crate::proc::process_started_at(pid);
1692        state.driver_exited = false;
1693        state.driver_claimed = Some(true);
1694        state.begin_pass();
1695        execution.driver.bind(state);
1696        state.save()?;
1697        execution.driver.record(state)?;
1698        Ok(execution)
1699    }
1700
1701    /// A grant owns only these repair attempts. Confirmation reviews cannot
1702    /// create another fixer allowance, and every operation is restart-checkpointed.
1703    async fn execute_granted_repair(&mut self) -> Result<()> {
1704        use crate::repair_budget::{Kind, Phase};
1705        let index = self.state.repair_grants.len() - 1;
1706        let winner = self
1707            .state
1708            .winner()
1709            .cloned()
1710            .context("repair winner missing")?;
1711        loop {
1712            let grant = self.state.repair_grants[index].clone();
1713            match grant.phase {
1714                Phase::Pending => {
1715                    if grant.spent >= grant.receipt.request.rounds {
1716                        self.state.repair_grants[index].phase = Phase::Complete;
1717                        self.state.status = RunStatus::Blocked;
1718                        self.state.save()?;
1719                        return Ok(());
1720                    }
1721                    // Reserve before dispatch. A crash can consume an attempt,
1722                    // but can never duplicate it or reset saved fixer history.
1723                    self.state.repair_grants[index].spent += 1;
1724                    self.state.repair_grants[index].phase = Phase::Fixing;
1725                    self.state.save()?;
1726                    match grant.receipt.request.kind {
1727                        Kind::Gate => {
1728                            let failed = self.state.gate.clone();
1729                            let _ = self.gate_fix_round(&winner, &failed).await?;
1730                        }
1731                        Kind::Review => {
1732                            let mut record = self
1733                                .state
1734                                .reviews
1735                                .last()
1736                                .cloned()
1737                                .context("review failure missing")?;
1738                            let round = record.round;
1739                            let step = self.run_fix_round(&winner, &mut record).await?;
1740                            self.state.reviews[round - 1] = record;
1741                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1742                                self.state.repair_grants[index].phase = Phase::Complete;
1743                                self.state.status = RunStatus::Blocked;
1744                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1745                                self.state.save()?;
1746                                return Ok(());
1747                            }
1748                            if matches!(step, FixStep::Resource) {
1749                                self.stop_for_resource()?;
1750                                return Ok(());
1751                            }
1752                            self.state.save()?;
1753                        }
1754                    }
1755                    if self.resource_halt()? {
1756                        return Ok(());
1757                    }
1758                }
1759                Phase::Fixing => {
1760                    // Interrupted fixers may have left source changes or an
1761                    // unpushed commit. Preserve them before fresh assessment.
1762                    let _coordination = self.coordinate(&winner.worktree).await?;
1763                    let rescued = git::rescue_commit(
1764                        &winner.worktree,
1765                        "magi: preserve interrupted granted repair",
1766                    )
1767                    .await?;
1768                    self.state.note_withheld("repair-grant", &rescued.withheld);
1769                    // Fresh checks must supersede even a same-head failure.
1770                    // Archive rather than erase failure evidence or counters.
1771                    self.state
1772                        .verification_history
1773                        .push(crate::run::VerificationEvidence {
1774                            superseded_by: None,
1775                            archived_at: Some(Timestamp::now()),
1776                            fixes_spent: Some(self.state.gate_fixes.len()),
1777                            gate_head: self.state.gate_head.take(),
1778                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1779                            gate: std::mem::take(&mut self.state.gate),
1780                            e2e: self.state.head_e2e.take(),
1781                        });
1782                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1783                        self.state
1784                            .verification_history
1785                            .push(crate::run::VerificationEvidence {
1786                                superseded_by: None,
1787                                archived_at: Some(Timestamp::now()),
1788                                fixes_spent: Some(self.state.gate_fixes.len()),
1789                                gate_head: None,
1790                                gate_ran: false,
1791                                gate: Vec::new(),
1792                                e2e: Some(e2e),
1793                            });
1794                    }
1795                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1796                    self.state.repair_grants[index].head =
1797                        git::rev_parse(&winner.worktree, "HEAD").await?;
1798                    self.state.repair_grants[index].phase = Phase::Reviewing;
1799                    self.state.save()?;
1800                }
1801                Phase::Reviewing => {
1802                    self.review_loop_pass().await?;
1803                    if self.resource_halt()? {
1804                        return Ok(());
1805                    }
1806                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1807                    let clean = self.state.reviews.len() > grant.review_start
1808                        && self.state.reviews.last().is_some_and(|r| {
1809                            r.head == head
1810                                && r.clean
1811                                && r.expected > 0
1812                                && r.answered == r.expected
1813                                && matches!(
1814                                    r.e2e_status(),
1815                                    E2eStatus::Passed | E2eStatus::NotConfigured
1816                                )
1817                        });
1818                    if !clean {
1819                        // A confirmation review carries no additional fix budget.
1820                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1821                            == Kind::Review
1822                            && grant.spent < grant.receipt.request.rounds
1823                            && self.state.reviews.last().is_some_and(|r| {
1824                                r.fix.is_none()
1825                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1826                            }) {
1827                            Phase::Pending
1828                        } else {
1829                            Phase::Complete
1830                        };
1831                        self.state.status = RunStatus::Blocked;
1832                        self.state.event(
1833                            "repair-grant",
1834                            "fresh review did not clear the repaired head; no publication",
1835                        );
1836                        self.state.save()?;
1837                        if self.state.repair_grants[index].phase == Phase::Pending {
1838                            continue;
1839                        }
1840                        return Ok(());
1841                    }
1842                    self.state.repair_grants[index].phase = Phase::Gating;
1843                    self.state.save()?;
1844                }
1845                Phase::Gating => {
1846                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1847                    self.state.reconcile_verification_head(&head);
1848                    let wait = self.repository_wait();
1849                    if self
1850                        .state
1851                        .reviews
1852                        .last()
1853                        .is_none_or(|r| r.head != head || !r.clean)
1854                    {
1855                        self.state.status = RunStatus::Blocked;
1856                        self.state.repair_grants[index].phase = Phase::Complete;
1857                        self.state.event(
1858                            "repair-grant",
1859                            "head changed since fresh review; repair held",
1860                        );
1861                        self.state.save()?;
1862                        return Ok(());
1863                    }
1864                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1865                        return Ok(());
1866                    }
1867                    let outcomes = self.run_gate(&winner).await?;
1868                    if verify_inconclusive(&outcomes) {
1869                        return Ok(());
1870                    }
1871                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1872                    if after != head {
1873                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1874                        self.state.repair_grants[index].head = after;
1875                        self.state.repair_grants[index].phase = Phase::Reviewing;
1876                        self.state.status = RunStatus::Blocked;
1877                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1878                        self.state.save()?;
1879                        return Ok(());
1880                    }
1881                    let passed = outcomes.iter().all(CommandOutcome::ok);
1882                    self.state.repair_grants[index].succeeded = passed;
1883                    self.state.gate = outcomes;
1884                    self.state.gate_ran = true;
1885                    self.state.status = if passed {
1886                        RunStatus::Gating
1887                    } else {
1888                        RunStatus::Blocked
1889                    };
1890                    self.state.repair_grants[index].phase = if !passed
1891                        && grant.receipt.request.kind == Kind::Gate
1892                        && grant.spent < grant.receipt.request.rounds
1893                    {
1894                        Phase::Pending
1895                    } else {
1896                        Phase::Complete
1897                    };
1898                    if passed && self.state.config.merge.mode == MergeMode::Pr {
1899                        // Persist publication debt with the successful checkpoint.
1900                        self.state.pr_publication_pending = true;
1901                    } else if passed {
1902                        self.state.status = RunStatus::Ready;
1903                    }
1904                    self.state.save()?;
1905                    if !passed {
1906                        continue;
1907                    }
1908                    // Existing PR ownership and push lease stay in place. The
1909                    // normal publisher owes the new reviewed/gated SHA, and
1910                    // land's fresh owner approval remains mandatory.
1911                    if self.state.config.merge.mode != MergeMode::Pr {
1912                        self.state.status = RunStatus::Ready;
1913                        self.state.event(
1914                            "repair-grant",
1915                            "repair verified locally; grant carries no merge authorization",
1916                        );
1917                        self.state.save()?;
1918                        return Ok(());
1919                    }
1920                    return self.merge().await;
1921                }
1922                Phase::Complete => return Ok(()),
1923            }
1924        }
1925    }
1926
1927    async fn execute_graph(&mut self) -> Result<()> {
1928        // Even a terminal run may reenter through an explicit recovery action.
1929        // Restore before prep shortcuts, saved conversations or graph nodes use
1930        // a disposable path; GC may already have removed its original objects.
1931        let archive_home = crate::run::home().canonicalize()?;
1932        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1933            let repo = self.state.repo.clone();
1934            let _coordination = self.coordinate(&repo).await?;
1935            ensure_setup_config(&mut self.state, &repo).await?;
1936            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1937        }
1938        self.state.configure_recovery_resume()?;
1939        if self
1940            .state
1941            .repair_grants
1942            .last()
1943            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1944        {
1945            return self.execute_granted_repair().await;
1946        }
1947        if self
1948            .state
1949            .repair_grants
1950            .last()
1951            .is_some_and(|g| !g.succeeded)
1952        {
1953            self.state.status = RunStatus::Blocked;
1954            self.state.save()?;
1955            return Ok(());
1956        }
1957        // Preservation may invalidate an old empty classification. Resolve it
1958        // before any resume shortcut can reuse a verdict or publish a branch.
1959        self.persist_candidates().await?;
1960        // A saved base stop outranks stage markers, including legacy Voting.
1961        // Recovery never reenters implementation, deliberation or voting.
1962        if self.state.base_sync_blocked() {
1963            self.sync_to_base().await?;
1964            if !self.state.base_sync_blocked()
1965                && self.state.status != RunStatus::AlreadyInBase
1966                && !self.resource_halt()?
1967            {
1968                self.finish_after_tally().await?;
1969            }
1970            return Ok(());
1971        }
1972        if self
1973            .state
1974            .merge_decision
1975            .as_ref()
1976            .is_some_and(|decision| !decision.closed)
1977            && self.state.status != RunStatus::Landing
1978        {
1979            if crate::merge_decision::resumable_hold(&self.state) {
1980                self.state.merge = Some(MergeOutcome {
1981                    mode: MergeMode::Pr,
1982                    ok: false,
1983                    detail: self
1984                        .state
1985                        .merge_decision
1986                        .as_ref()
1987                        .unwrap()
1988                        .target
1989                        .pr
1990                        .clone(),
1991                    empty: false,
1992                });
1993                self.run_land().await?;
1994                self.settle_questions();
1995                return Ok(());
1996            }
1997            bail!(
1998                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1999            );
2000        }
2001        if self.state.config.graph.selected_implementer.is_some()
2002            && self.state.status == RunStatus::Blocked
2003            && self.state.tally.is_none()
2004            && self.state.candidate_preservation.is_empty()
2005            && self.state.interrupted_implementations.is_empty()
2006        {
2007            bail!(
2008                "selected implementation is held; explicitly release the owning task to retry it"
2009            );
2010        }
2011
2012        if self.state.merge_repairs.last().is_some_and(|r| {
2013            matches!(
2014                r.phase,
2015                crate::run::MergeRepairPhase::Requested
2016                    | crate::run::MergeRepairPhase::Fixing
2017                    | crate::run::MergeRepairPhase::Reviewing
2018            )
2019        }) {
2020            if self.repair_merge_review().await? {
2021                self.run_land().await?;
2022            }
2023            self.settle_questions();
2024            return Ok(());
2025        }
2026        // Publication already passed review and gate. Resume only that node:
2027        // syncing to a newer base here could change the verified commit.
2028        if self.state.pr_publication_pending {
2029            self.merge().await?;
2030            if self.resource_halt()? {
2031                return Ok(());
2032            }
2033            self.state.save()?;
2034            self.settle_questions();
2035            return Ok(());
2036        }
2037        // A run that already lost its quorum never resumes into the verdict
2038        // machinery: `deliberate` and `vote` would otherwise clobber the
2039        // stalled marker back to Voting and the run would keep going past a
2040        // verdict that is no longer trustworthy. Everything already recorded is
2041        // kept, so the run stays resumable (or foldable) for a human to pick up.
2042        //
2043        // On --resume the run gets one chance to repair itself: the seats a
2044        // rate limit took out are re-asked. If their quota has since reset and
2045        // the quorum is restored, the run picks up and finishes; otherwise it
2046        // stays stale and still-resumable for a later retry. If it does not
2047        // recover, the returned status stays `Stalled` and nothing was
2048        // clobbered (the recovery only mutates entries for the lost seats).
2049        if self.state.status == RunStatus::Stalled {
2050            if self.recover_stall().await? {
2051                self.finish_after_tally().await?;
2052            } else {
2053                // Still below quorum: persist the marker and stay resumable.
2054                self.state.save()?;
2055            }
2056            return Ok(());
2057        }
2058        // A run parked inside `land` - watching CI, mid fix-round, or
2059        // waiting on the owner's merge approval - resumes directly into it,
2060        // never back through `prep`. Everything before `merge` already
2061        // concluded; that is the only way `status` reaches `Landing` in the
2062        // first place. Re-walking `review_loop` first would also be actively
2063        // wrong: its own status recomputation (see its doc) treats any
2064        // clean round as reason to set `status` to `Gating`, which would
2065        // clobber this marker before `merge` ever ran, and this run would
2066        // never find its way back into `land` at all.
2067        if self.state.status == RunStatus::Landing
2068            || crate::handover::landing_checkpoint(&self.state)
2069        {
2070            self.state.land_sync_pending = true;
2071            self.state.save()?;
2072            self.run_land().await?;
2073            // `run_land` may have settled the run right here - CI came back
2074            // green and the PR merged, say - without ever passing back
2075            // through `merge`'s own trailing call. Whatever it left `status`
2076            // as is what this has to read.
2077            self.settle_questions();
2078            return Ok(());
2079        }
2080        // A run parked at `merge` on the owner's word about a withheld PR text
2081        // resumes straight into `merge`. Re-walking the tail would sync to the
2082        // base again (possibly rebasing, possibly spending the fixer) after the
2083        // gate had passed, and then push a tree nobody reviewed or gated.
2084        if self.state.github_text.is_some()
2085            && self.state.merge.is_none()
2086            && self.state.status == RunStatus::Gating
2087        {
2088            self.merge().await?;
2089            if self.resource_halt()? {
2090                return Ok(());
2091            }
2092            self.state.save()?;
2093            return Ok(());
2094        }
2095        if let Err(e) = self.prep().await {
2096            if e.is::<crate::repo_lock::ResourceBlocked>() {
2097                return Err(e);
2098            }
2099            self.state.status = RunStatus::Blocked;
2100            self.state
2101                .prep_failure
2102                .get_or_insert_with(|| format!("{e:#}"));
2103            self.state
2104                .event("prep", format!("preparation stopped: {e:#}"));
2105            self.state.save()?;
2106            return Err(e);
2107        }
2108        if self.park_here()? {
2109            return Ok(());
2110        }
2111        self.advise().await?;
2112        if self.park_here()? {
2113            return Ok(());
2114        }
2115        self.implement().await?;
2116        if self.park_here()? {
2117            return Ok(());
2118        }
2119        // `after_implement` already saved the state and settled any open
2120        // questions when it set this; nothing later in the graph has
2121        // anything to judge.
2122        if self.state.status == RunStatus::VerifiedNoop {
2123            return Ok(());
2124        }
2125        self.judge().await?;
2126        if self.park_here()? {
2127            return Ok(());
2128        }
2129        self.deliberate().await?;
2130        if self.park_here()? {
2131            return Ok(());
2132        }
2133        self.vote().await?;
2134        if self.park_here()? {
2135            return Ok(());
2136        }
2137        self.tally()?;
2138        // A verdict that lost its quorum is not trustworthy: do not review,
2139        // gate, or merge on it. Everything already done is kept, so the run
2140        // stays resumable (or foldable); the human can replace the agent that
2141        // ran out of quota and pick it up.
2142        if self.state.status == RunStatus::Stalled {
2143            // Persist the stalled marker now — the normal end-of-execute save
2144            // below is below this early return, and without it a resumed run
2145            // would reload a pre-tally status and keep going.
2146            self.state.save()?;
2147            return Ok(());
2148        }
2149        self.finish_after_tally().await?;
2150        Ok(())
2151    }
2152
2153    /// A wave could not start its agents because a shared resource stayed busy
2154    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2155    /// resumable. Unstarted seats have no result; completed peers retain
2156    /// their saved candidate state.
2157    fn stop_for_resource(&mut self) -> Result<()> {
2158        // Resource acquisition observes cancellation inside a node. Complete
2159        // the same durable park as every other node boundary before returning.
2160        if self.pause.parked() || self.interrupt.parked() {
2161            self.park_here()?;
2162            return Ok(());
2163        }
2164        if self.state.resource_block.is_some() {
2165            self.state.status = RunStatus::Blocked;
2166        }
2167        self.state.save()
2168    }
2169
2170    /// Stop walking the graph if a node ended on a resource block.
2171    fn resource_halt(&mut self) -> Result<bool> {
2172        if self.state.resource_block.is_none() {
2173            return Ok(false);
2174        }
2175        self.stop_for_resource()?;
2176        Ok(true)
2177    }
2178
2179    /// Park here if asked to, recording it in the run's own timeline.
2180    ///
2181    /// Returns whether the caller should stop walking the graph. The state is
2182    /// saved either way by the node that just finished; this adds the event so
2183    /// the operator's card says why a run that is neither finished nor moving
2184    /// is sitting where it is.
2185    fn park_here(&mut self) -> Result<bool> {
2186        let parking = self.pause.parked() || self.interrupt.parked();
2187        if !parking && self.state.resource_block.is_some() {
2188            self.stop_for_resource()?;
2189            return Ok(true);
2190        }
2191        // Either handle asking is enough - see `Pause`'s own doc for why
2192        // they are never the same one. `interrupt` is checked second so a
2193        // reason it carries is preferred in the message below over a plain
2194        // shutdown park racing it at the same boundary.
2195        if !self.pause.parked() && !self.interrupt.parked() {
2196            return Ok(false);
2197        }
2198        if self.state.parked {
2199            return Ok(true);
2200        }
2201        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2202        if let Some(block) = &mut self.state.resource_block
2203            && matches!(
2204                block.cause,
2205                crate::run::ResourceBlockCause::Cancelled { .. }
2206            )
2207        {
2208            block.cause = crate::run::ResourceBlockCause::Cancelled {
2209                park: cause.clone(),
2210            };
2211        }
2212        let why = match cause.as_ref().map(ParkCause::label) {
2213            Some(reason) => format!(
2214                "parked after `{}` ({reason}) — resume to carry on from here",
2215                self.state.status.as_str()
2216            ),
2217            None => format!(
2218                "parked after `{}` — resume to carry on from here",
2219                self.state.status.as_str()
2220            ),
2221        };
2222        self.state.event("park", why);
2223        self.state
2224            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2225        self.state.parked = true;
2226        self.state.save()?;
2227        Ok(true)
2228    }
2229
2230    /// Only a new explicit operator approval may retry failed preparation.
2231    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2232        if self.state.prep_failure.is_none() {
2233            return Ok(());
2234        }
2235        let failed_at = self
2236            .state
2237            .events
2238            .iter()
2239            .rev()
2240            .find(|event| event.node == "prep")
2241            .map_or(self.state.updated_at, |event| event.at);
2242        anyhow::ensure!(
2243            approval_at > failed_at,
2244            "preparation retry requires a new explicit release after the failure"
2245        );
2246        self.state.prep_failure = None;
2247        self.state.prep_complete = Some(false);
2248        self.state
2249            .event("prep-retry", "operator approved retry of preparation");
2250        Ok(())
2251    }
2252
2253    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2254    pub fn on_pause(&mut self, pause: Pause) {
2255        self.pause = pause;
2256    }
2257
2258    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2259    /// scheduler asking this one run - and no other - to park so a task
2260    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2261    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2262    /// handle.
2263    pub fn watch_interrupt(&mut self, pause: Pause) {
2264        self.interrupt = pause;
2265    }
2266
2267    /// Abandon this run's own open questions, once `status` has actually
2268    /// settled rather than merely paused.
2269    ///
2270    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2271    /// either back up with the candidates, the review round and the seat
2272    /// sessions already on disk, so a question an implementer asked mid-round
2273    /// may still get a real answer read by a real resume. Only the statuses
2274    /// `resumable` excludes are actually final: the run merged, it reached
2275    /// `Ready` with nothing left to do, it failed outright with no
2276    /// established point to continue from, or every candidate agreed, with
2277    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2278    /// every one of those the seat that asked is gone for good, exactly like
2279    /// the run being deleted under `magi run rm` - so the same cleanup
2280    /// applies, worded for what actually happened instead of "the run was
2281    /// deleted".
2282    ///
2283    /// Best-effort and silent on success: called from every place `status`
2284    /// can land on one of those three, including ones a resumed run revisits,
2285    /// so it must cost nothing when there was nothing open to begin with.
2286    fn settle_questions(&mut self) {
2287        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2288            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2289        }
2290    }
2291
2292    /// The tail of the graph after a trustworthy tally: fold losers, review,
2293    /// gate, merge, and persist.
2294    async fn finish_after_tally(&mut self) -> Result<()> {
2295        // Before the losers are folded: their branches are what the graft
2296        // reads from.
2297        self.graft().await?;
2298        if self.resource_halt()? {
2299            return Ok(());
2300        }
2301        self.fold_losers().await?;
2302        // Before review starts, and again right before the gate: a run's
2303        // review rounds can themselves take long enough for the base to move
2304        // a second time, and the gate is the one node whose "green" gets
2305        // acted on.
2306        self.sync_to_base().await?;
2307        if self.hold_base_sync().await? || self.resource_halt()? {
2308            return Ok(());
2309        }
2310        if self.state.status == RunStatus::AlreadyInBase {
2311            return Ok(());
2312        }
2313        self.review_loop().await?;
2314        if self.resource_halt()? {
2315            return Ok(());
2316        }
2317        self.sync_to_base().await?;
2318        if self.hold_base_sync().await? || self.resource_halt()? {
2319            return Ok(());
2320        }
2321        if self.state.status == RunStatus::AlreadyInBase {
2322            return Ok(());
2323        }
2324        self.gate().await?;
2325        self.merge().await?;
2326        if self.resource_halt()? {
2327            return Ok(());
2328        }
2329        self.state.save()?;
2330        Ok(())
2331    }
2332
2333    // ---------------------------------------------------------------- prep
2334
2335    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2336    /// run with the step and its output in the event; it is never swallowed.
2337    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2338    /// resume starts over instead of finding candidates and missing seats.
2339    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2340        let result = match ensure_setup_config(&mut self.state, repo).await {
2341            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2342            Err(e) => Err(e),
2343        };
2344        if let Err(e) = result {
2345            git::worktree_remove_clean(repo, wt).await.ok();
2346            if in_prep {
2347                self.rollback_prep(repo).await;
2348            }
2349            self.state.status = RunStatus::Blocked;
2350            self.state.event(
2351                "setup",
2352                format!("worktree setup failed in {}: {e:#}", wt.display()),
2353            );
2354            self.state.save()?;
2355            return Err(e);
2356        }
2357        Ok(())
2358    }
2359
2360    /// Undo everything `prep` created, so a resume re-runs it whole.
2361    async fn rollback_prep(&mut self, repo: &Path) {
2362        let root = self.state.worktree_root();
2363        for c in std::mem::take(&mut self.state.candidates) {
2364            if matches!(
2365                git::worktree_remove_clean(repo, &c.worktree).await,
2366                Ok(true)
2367            ) {
2368                git::branch_delete(repo, &c.branch).await.ok();
2369            } else {
2370                self.state.candidates.push(c);
2371            }
2372        }
2373        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2374        for j in 1..=seats {
2375            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2376                .await
2377                .ok();
2378        }
2379        for k in 1..=self.state.config.graph.advisors {
2380            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2381                .await
2382                .ok();
2383        }
2384    }
2385
2386    async fn prep(&mut self) -> Result<()> {
2387        if let Some(reason) = &self.state.prep_failure {
2388            bail!(
2389                "candidate preparation previously failed: {reason}; correct the source before retrying"
2390            );
2391        }
2392        if self
2393            .state
2394            .prep_complete
2395            .unwrap_or(!self.state.candidates.is_empty())
2396        {
2397            return Ok(());
2398        }
2399        let repo = self.state.repo.clone();
2400        // Includes rollback, config, refs, checkout creation and setup. No
2401        // repository mutation may precede the real shared lock acquisition.
2402        let _coordination = self.coordinate(&repo).await?;
2403        let base = self.state.base_commit.clone();
2404        let plan = self.state.sources.plan(&repo, &base).await?;
2405        self.rollback_prep(&repo).await;
2406        if !self.state.candidates.is_empty() {
2407            bail!(
2408                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2409            );
2410        }
2411        self.state.prep_complete = Some(false);
2412        self.state.status = RunStatus::Prep;
2413        self.state.save()?;
2414        let repo = self.state.repo.clone();
2415        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2416        let root = self.state.worktree_root();
2417        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2418
2419        // The hook is the write-time half of the blindness contract; the
2420        // presentation filter in `blind` is the half that cannot be bypassed.
2421        let hooks_dir = self.state.dir().join("hooks");
2422        if self.state.config.blind.commit_msg_hook {
2423            std::fs::create_dir_all(&hooks_dir)
2424                .with_context(|| format!("create {}", hooks_dir.display()))?;
2425            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2426            let path = hooks_dir.join("commit-msg");
2427            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2428            make_executable(&path)?;
2429            // Ref-counted rather than a plain idempotent set: with more than
2430            // one run able to be in flight in the same repository at once
2431            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2432            // true?" check cannot tell "another run of mine still needs
2433            // this" from "nobody does", and the run that happens to finish
2434            // first would disable the hook out from under a sibling still
2435            // relying on it.
2436            git::acquire_worktree_config(&repo).await?;
2437            self.state.enabled_worktree_config = true;
2438        }
2439
2440        // `[worktree] setup` hides its products per worktree, which needs the
2441        // same per-worktree config the hook does. Held until fold, like it.
2442        ensure_setup_config(&mut self.state, &repo).await?;
2443
2444        for (index, (spec, label)) in self
2445            .roles
2446            .implementers
2447            .clone()
2448            .into_iter()
2449            .zip(labels)
2450            .enumerate()
2451        {
2452            let branch = self.state.branch_for(label);
2453            let worktree = root.join(format!("cand-{label}"));
2454            // Persist intent before Git can leave a partially created tree.
2455            // Candidate registration follows setup, so it cannot inventory
2456            // crashes or refusals during creation itself.
2457            self.state.event(
2458                "prep-intent",
2459                serde_json::json!({
2460                    "branch": branch,
2461                    "worktree": worktree,
2462                    "start": start,
2463                    "picks": plan.picks,
2464                })
2465                .to_string(),
2466            );
2467            self.state.save()?;
2468            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2469            if self.state.config.blind.commit_msg_hook {
2470                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2471            }
2472            git::local_exclude(&worktree, "/.magi/").await?;
2473            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2474                // Nothing of this prep may survive: a resume must start over,
2475                // not find half the candidates and skip the rest.
2476                git::worktree_remove_clean(&repo, &worktree).await.ok();
2477                git::branch_delete(&repo, &branch).await.ok();
2478                self.rollback_prep(&repo).await;
2479                self.state.status = RunStatus::Blocked;
2480                self.state
2481                    .event("prep", format!("worktree setup failed: {e:#}"));
2482                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2483                self.state.save()?;
2484                return Err(e);
2485            }
2486            self.state.candidates.push(Candidate {
2487                index,
2488                label,
2489                agent: spec.id.clone(),
2490                branch,
2491                worktree,
2492                summary: String::new(),
2493                stat: String::new(),
2494                files: 0,
2495                commits: 0,
2496                empty: false,
2497                failed: None,
2498                verified_noop: None,
2499                duration_ms: 0,
2500                folded: false,
2501            });
2502            self.state.save()?;
2503            for pick in &plan.picks {
2504                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2505                {
2506                    self.state.status = RunStatus::Blocked;
2507                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2508                    self.state
2509                        .event("prep", format!("cannot apply referenced commit: {e}"));
2510                    self.state.save()?;
2511                    return Err(e);
2512                }
2513            }
2514        }
2515
2516        for j in 1..=self.roles.judges.len() {
2517            let wt = root.join(format!("judge-{j}"));
2518            if !wt.exists() {
2519                git::worktree_add_detached(&repo, &wt, &base).await?;
2520                self.setup_seat_worktree(&repo, &wt, true).await?;
2521            }
2522        }
2523
2524        // Disposable, detached checkouts for the design-deliberation stage's
2525        // advisor seats — the same shape as the judges' above, at the same
2526        // base commit, since advisors also only ever read. Sized off the
2527        // configured count directly rather than a resolved roster: unlike
2528        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2529        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2530        // `prep` has no `ResolvedRoles` field to read a count from here.
2531        if self.state.config.graph.advise {
2532            for k in 1..=self.state.config.graph.advisors {
2533                let wt = root.join(format!("advisor-{k}"));
2534                if !wt.exists() {
2535                    git::worktree_add_detached(&repo, &wt, &base).await?;
2536                    self.setup_seat_worktree(&repo, &wt, true).await?;
2537                }
2538            }
2539        }
2540
2541        // A judge cannot tell it is looking at its own patch — the seats keep
2542        // separate conversations — but a panel that shares agents with the
2543        // field is less independent than it looks, and that is worth saying out
2544        // loud once per run rather than leaving it in the config.
2545        let authors: Vec<&str> = self
2546            .roles
2547            .implementers
2548            .iter()
2549            .map(|a| a.id.as_str())
2550            .collect();
2551        let overlap: Vec<String> = self
2552            .roles
2553            .judges
2554            .iter()
2555            .enumerate()
2556            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2557            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2558            .collect();
2559        if !overlap.is_empty() {
2560            let note = format!(
2561                "{} also authored a candidate; blind, but the panel is less \
2562                 independent than {} distinct agents would be",
2563                overlap.join(", "),
2564                self.roles.judges.len()
2565            );
2566            self.state.event("prep", note);
2567        }
2568
2569        self.state.event(
2570            "prep",
2571            format!(
2572                "{} candidates, {} judges, base {} ({})",
2573                self.state.candidates.len(),
2574                self.roles.judges.len(),
2575                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2576                self.state.base_branch
2577            ),
2578        );
2579        self.state.prep_complete = Some(true);
2580        self.state.status = RunStatus::Implementing;
2581        self.state.save()?;
2582        Ok(())
2583    }
2584
2585    // -------------------------------------------------------------- advise
2586
2587    /// The design-deliberation stage: independent, read-only advisor seats
2588    /// each sketch a design before any implementer touches the repository,
2589    /// and (when at least one produced a usable proposal) a synthesis seat
2590    /// blends them into a brief `implement` carries in every candidate's
2591    /// prompt.
2592    ///
2593    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2594    /// advisors` is the proposal count. Everything here is best-effort and
2595    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2596    /// that cannot reach quota, or a synthesis seat that produced nothing
2597    /// usable all leave `implement` exactly as it was before this stage
2598    /// existed — the task instruction alone — rather than failing the whole
2599    /// competition over an enrichment stage. Every outcome is still recorded
2600    /// as an event, so a run that got nothing from this stage says why.
2601    ///
2602    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2603    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2604    /// resumed run whose stage failed would re-run it, and re-spend the
2605    /// agent calls, on every reentry before `implement`.
2606    ///
2607    /// Also skipped once any candidate shows implementation progress — the
2608    /// exact predicate `implement` itself uses to decide a candidate is no
2609    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2610    /// is not enough: a run created by an older binary that predates this
2611    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2612    /// an already-`Implementing`-or-later run under this build would
2613    /// otherwise walk straight back through `prep` (a no-op once candidates
2614    /// exist) into this node and spawn every advisor seat against worktrees
2615    /// `prep` never recreated — after implementation has already started,
2616    /// which is exactly the invariant this stage exists to guarantee.
2617    async fn advise(&mut self) -> Result<()> {
2618        let implement_untouched = self
2619            .state
2620            .candidates
2621            .iter()
2622            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2623        if !self.state.config.graph.advise || self.state.advise_attempted {
2624            return Ok(());
2625        }
2626        if !implement_untouched {
2627            self.state.event(
2628                "advise",
2629                "skipping the design-deliberation stage: at least one \
2630                 candidate already shows implementation progress, so this \
2631                 run is past the point the stage exists to run before"
2632                    .to_owned(),
2633            );
2634            self.state.advise_attempted = true;
2635            self.state.save()?;
2636            return Ok(());
2637        }
2638        let run_id = self.state.id.clone();
2639        let prompts = self.state.config.prompts.clone();
2640        let instruction = self.state.instruction.clone();
2641        let language = self.state.config.graph.language.clone();
2642        let root = self.state.worktree_root();
2643        let n = self.state.config.graph.advisors;
2644        let where_recorded = self.state.dir().join("run.json");
2645
2646        let seats = match self.state.config.advisors() {
2647            Ok(seats) if !seats.is_empty() => seats,
2648            Ok(_) => {
2649                self.state.event(
2650                    "advise",
2651                    format!(
2652                        "[graph] advisors is 0; skipping the design-deliberation \
2653                         stage and continuing without a synthesis brief (see {})",
2654                        where_recorded.display()
2655                    ),
2656                );
2657                self.state.advise_attempted = true;
2658                self.state.save()?;
2659                return Ok(());
2660            }
2661            Err(e) => {
2662                self.state.event(
2663                    "advise",
2664                    format!(
2665                        "could not resolve advisor seats ({e:#}); continuing \
2666                         without a design-deliberation brief (see {})",
2667                        where_recorded.display()
2668                    ),
2669                );
2670                self.state.advise_attempted = true;
2671                self.state.save()?;
2672                return Ok(());
2673            }
2674        };
2675
2676        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2677        let artifacts = agent::artifacts_dir(&self.state.dir());
2678        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2679
2680        let mut jobs = Vec::new();
2681        for (i, spec) in seats.iter().cloned().enumerate() {
2682            let seat_key = format!("advisor-{}", i + 1);
2683            let seat = self.seat(&seat_key, &spec.id);
2684            jobs.push(SeatJob {
2685                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2686                spec,
2687                seat,
2688                cwd: worktrees[i % worktrees.len()].clone(),
2689                timeout,
2690                allow_write: false,
2691                sessions: false,
2692                artifacts: artifacts.clone(),
2693                stem: seat_key,
2694                handover: None,
2695            });
2696        }
2697
2698        self.state.event(
2699            "advise",
2700            format!(
2701                "{} advisor seat(s) sketching a design in parallel",
2702                jobs.len()
2703            ),
2704        );
2705        let mut quota_losses = Vec::new();
2706        let cache = self.state.config.cache_dir();
2707        let ctx = WaveCtx {
2708            carry_seats: false,
2709            pauses: Some((&self.pause, &self.interrupt)),
2710            run: &run_id,
2711            node: "advise",
2712            prompts: &prompts,
2713            cache: cache.as_deref(),
2714            round: None,
2715        };
2716        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2717        let results = ask_json_wave::<Proposal>(
2718            jobs,
2719            Arc::clone(&self.sem),
2720            self.state.config.graph.retries,
2721            &advisor_roster,
2722            &ctx,
2723            &mut quota_losses,
2724            &mut self.state,
2725            &|p: &Proposal| p.validate(),
2726        )
2727        .await;
2728        if self.state.resource_block.is_some() {
2729            return self.stop_for_resource();
2730        }
2731        self.state.quota.extend(quota_losses);
2732
2733        let mut records = Vec::with_capacity(results.len());
2734        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2735            let agent_id = seat.agent.clone();
2736            self.state.seats.insert(seat.key.clone(), seat);
2737            match res {
2738                Ok((proposal, out)) => {
2739                    self.state
2740                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2741                    records.push(advise::AdvisorRecord::proposed(
2742                        i + 1,
2743                        agent_id,
2744                        proposal,
2745                        out.duration_ms,
2746                    ));
2747                }
2748                Err(e) => {
2749                    self.state.event(
2750                        "advise",
2751                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2752                    );
2753                    records.push(advise::AdvisorRecord::failed(
2754                        i + 1,
2755                        agent_id,
2756                        e.to_string(),
2757                    ));
2758                }
2759            }
2760        }
2761
2762        let mut advice = advise::Advice {
2763            records,
2764            synthesis: None,
2765        };
2766        if advice.proposals().is_empty() {
2767            self.state.event(
2768                "advise",
2769                "no advisor produced a usable proposal; continuing without a \
2770                 synthesis brief"
2771                    .to_owned(),
2772            );
2773        } else {
2774            match self
2775                .synthesize_brief(
2776                    &advice,
2777                    &instruction,
2778                    &language,
2779                    &worktrees[0],
2780                    &artifacts,
2781                    &run_id,
2782                    &prompts,
2783                    cache.as_deref(),
2784                )
2785                .await
2786            {
2787                Ok(Some(text)) => {
2788                    self.state.event(
2789                        "advise",
2790                        "synthesized a design brief for the implementer".to_owned(),
2791                    );
2792                    advice.synthesis = Some(text);
2793                }
2794                Ok(None) => {
2795                    self.state.event(
2796                        "advise",
2797                        "the synthesis seat produced nothing usable; continuing \
2798                         without a design brief"
2799                            .to_owned(),
2800                    );
2801                }
2802                Err(e) => {
2803                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2804                        return Err(e);
2805                    }
2806                    self.state.event(
2807                        "advise",
2808                        format!("could not synthesize a design brief: {e:#}"),
2809                    );
2810                }
2811            }
2812        }
2813        advise::apply_reflection(&mut advice);
2814
2815        self.state.advice = Some(advice);
2816        self.state.advise_attempted = true;
2817        self.state.save()?;
2818        Ok(())
2819    }
2820
2821    /// The synthesis seat: reads every advisor's proposal and blends them
2822    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2823    /// out of [`Runner::advise`] only for readability — it is not called
2824    /// anywhere else.
2825    ///
2826    /// Picked the same way [`crate::talk`]'s standing conversation and
2827    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2828    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2829    /// order (a claude seat, else the first runnable agent in roster order)
2830    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2831    /// in [`crate::config`] for why a dedicated field exists here at all.
2832    #[allow(clippy::too_many_arguments)]
2833    async fn synthesize_brief(
2834        &mut self,
2835        advice: &advise::Advice,
2836        instruction: &str,
2837        language: &str,
2838        cwd: &Path,
2839        artifacts: &Path,
2840        run_id: &str,
2841        prompts: &Prompts,
2842        cache: Option<&Path>,
2843    ) -> Result<Option<String>> {
2844        let chain = agent::pick_chain(
2845            &self.state.config.agents,
2846            self.state.config.roles.synthesizer.as_ref(),
2847            &agent::installed,
2848            "synthesizer",
2849        )?;
2850        let proposals = advice.proposals();
2851        let mut prompt = prompt::with_overlay(
2852            prompt::synthesize_brief(instruction, &proposals, language),
2853            prompts.overlay("advise"),
2854        );
2855        if cache.is_some() {
2856            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2857            // below — see `prompt::build_cache_note`'s doc for why telling a
2858            // read-only seat to build through the shared cache is exactly how
2859            // a sandbox's write refusal gets misread as a defect.
2860            prompt.push('\n');
2861            prompt.push_str(&prompt::build_cache_note("advise", false));
2862        }
2863        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2864        // Each id is tried once, in order; a quota hit, error or unusable
2865        // answer moves to the next. The seat is single-turn (`sessions:
2866        // false`) and the prompt is the whole context, so a fallback agent
2867        // needs nothing carried over.
2868        let mut last = None;
2869        for (n, spec) in chain.iter().enumerate() {
2870            if n > 0 {
2871                self.state
2872                    .event("advise", format!("synthesis falling back to {}", spec.id));
2873            }
2874            let mut seat = self.seat("advise-synthesis", &spec.id);
2875            let wait = self.repository_wait();
2876            let outcome = wait
2877                .scope(agent::invoke(
2878                    spec,
2879                    &mut seat,
2880                    &Invocation {
2881                        coordination: self.state.config.coordination.as_ref(),
2882                        fetched_base: None,
2883                        cwd,
2884                        prompt: &prompt,
2885                        timeout,
2886                        allow_write: false,
2887                        sessions: false,
2888                        artifacts,
2889                        stem: &if n == 0 {
2890                            "advise-synthesis".to_owned()
2891                        } else {
2892                            format!("advise-synthesis-{}", spec.id)
2893                        },
2894                        run: run_id,
2895                        node: "advise",
2896                        cache_dir: None,
2897                        attachments: &[],
2898                        writable: &[],
2899                    },
2900                ))
2901                .await;
2902            if outcome.is_ok() {
2903                self.state.seats.insert(seat.key.clone(), seat);
2904            }
2905            let advance = agent::chain_advances(&outcome);
2906            last = Some(outcome);
2907            if !advance {
2908                break;
2909            }
2910        }
2911        // Exhausted: the last attempt's result is what a single failed seat
2912        // would have produced.
2913        let out = last.expect("a chain holds at least one agent")?;
2914        if !out.usable() {
2915            return Ok(None);
2916        }
2917        let text =
2918            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2919        Ok((!text.trim().is_empty()).then_some(text))
2920    }
2921
2922    // ----------------------------------------------------------- implement
2923
2924    async fn implement(&mut self) -> Result<()> {
2925        self.persist_candidates().await?;
2926        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2927        // agent files with `magi task add` name the run that paid for it. The
2928        // prompt overlay is cloned alongside it because the waves borrow it
2929        // while `self` is mutably borrowed by the node's own bookkeeping.
2930        let run_id = self.state.id.clone();
2931        let prompts = self.state.config.prompts.clone();
2932        let todo: Vec<usize> = self
2933            .state
2934            .candidates
2935            .iter()
2936            .enumerate()
2937            .filter(|(_, c)| {
2938                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2939                    && c.failed.is_none()
2940                    && !c.empty
2941            })
2942            .map(|(i, _)| i)
2943            .collect();
2944        if todo.is_empty() {
2945            return self.after_implement();
2946        }
2947        self.state.status = RunStatus::Implementing;
2948
2949        let language = self.state.config.graph.language.clone();
2950        let github_language = self.state.config.graph.github_language.clone();
2951        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2952        let sessions = self.state.config.graph.sessions;
2953        let artifacts = agent::artifacts_dir(&self.state.dir());
2954        // The design-deliberation stage's blended brief, when `advise` found
2955        // one — carried into every implementer's prompt the same way
2956        // regardless of which candidate it is.
2957        let brief = self
2958            .state
2959            .advice
2960            .as_ref()
2961            .and_then(|a| a.synthesis.as_deref())
2962            .map(str::to_owned);
2963        let attachments = self.state.attachments.clone();
2964
2965        let mut jobs = Vec::new();
2966        for &i in &todo {
2967            let (index, label, worktree) = {
2968                let c = &self.state.candidates[i];
2969                (c.index, c.label, c.worktree.clone())
2970            };
2971            let spec = self.roles.implementers[index].clone();
2972            let seat_key = format!("impl-{label}");
2973            let seat = self.seat(&seat_key, &spec.id);
2974            let instruction = seeded_instruction(&self.state);
2975            jobs.push(SeatJob {
2976                spec,
2977                seat,
2978                prompt: prompt::implement_in(
2979                    &instruction,
2980                    &worktree.to_string_lossy(),
2981                    &language,
2982                    &github_language,
2983                    brief.as_deref(),
2984                    &attachments,
2985                ),
2986                cwd: worktree,
2987                timeout,
2988                allow_write: true,
2989                sessions,
2990                artifacts: artifacts.clone(),
2991                stem: format!("impl-{label}"),
2992                handover: None,
2993            });
2994        }
2995
2996        self.state.event(
2997            "implement",
2998            format!("{} candidates in parallel", jobs.len()),
2999        );
3000        // Kept so a seat whose CLI hung up can be asked again from the same
3001        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3002        // can update a seat's own entry once a fallback agent takes it over —
3003        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3004        // whichever agent actually answered, not the one that quota'd out.
3005        let mut sent = jobs.clone();
3006        let cache = self.state.config.cache_dir();
3007        let ctx = WaveCtx {
3008            carry_seats: false,
3009            pauses: Some((&self.pause, &self.interrupt)),
3010            run: &run_id,
3011            node: "implement",
3012            prompts: &prompts,
3013            cache: cache.as_deref(),
3014            round: None,
3015        };
3016        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3017        if !wave_blocked(&results) {
3018            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3019                .await;
3020            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3021                .await?;
3022            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3023                .await;
3024        }
3025        // Do not finalize failed candidates when the entire wave needs machine
3026        // recovery: leaving them pending resumes this node with its work intact.
3027        if !results.is_empty()
3028            && results
3029                .iter()
3030                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3031        {
3032            let losses: Vec<_> = results
3033                .iter()
3034                .map(|(_, seat, out)| {
3035                    let AgentOutcome::Quota(o) = out else {
3036                        unreachable!()
3037                    };
3038                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3039                })
3040                .collect();
3041            self.state.quota.extend(losses.clone());
3042            for (_, seat, _) in results {
3043                self.state.seats.insert(seat.key.clone(), seat);
3044            }
3045            for &i in &todo {
3046                let label = self.state.candidates[i].label;
3047                self.state.interrupted_implementations.insert(label);
3048                self.state
3049                    .candidate_preservation
3050                    .insert(label, "machine outage awaiting persistence".into());
3051            }
3052            // Preserve partial edits too, while keeping every interrupted seat
3053            // pending even when its tree already contains an inherited patch.
3054            block_machine_outage(&mut self.state, "implement", &losses);
3055            self.state.save()?;
3056            self.persist_candidates().await?;
3057            for &i in &todo {
3058                self.state
3059                    .interrupted_implementations
3060                    .insert(self.state.candidates[i].label);
3061            }
3062            return self.stop_for_resource();
3063        }
3064        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3065
3066        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3067            && results
3068                .iter()
3069                .any(|(_, _, out)| FailClass::of(out).is_some());
3070        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3071            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3072                // Executed turns interrupted by disk pressure already carry
3073                // a preservation barrier from wave. Admission-only waits do
3074                // not. Keep the interrupted seat's session for recovery too.
3075                if self
3076                    .state
3077                    .candidate_preservation
3078                    .contains_key(&self.state.candidates[i].label)
3079                {
3080                    self.state.seats.insert(seat.key.clone(), seat);
3081                }
3082                continue;
3083            }
3084            let seat_key = seat.key.clone();
3085            // A quota fallback (`resume_seat_handovers`) may have handed this
3086            // seat to a different agent than the one `prep` recorded on the
3087            // candidate; the stats tables and any later fixer-defaults-to-
3088            // winner's-author lookup must credit whoever actually answered —
3089            // unless every fallback also quota'd out, in which case nobody
3090            // actually answered and crediting the last agent tried would
3091            // erase every earlier agent's own quota loss from the stats
3092            // tables instead of just this one seat's.
3093            let agent = seat.agent.clone();
3094            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3095            self.state.seats.insert(seat.key.clone(), seat);
3096            let label = self.state.candidates[i].label;
3097            self.state.interrupted_implementations.remove(&label);
3098
3099            let (summary, duration, failed, verified_claim) = match out {
3100                AgentOutcome::Ok(o) => {
3101                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3102                    let failed = (!o.usable()).then(|| {
3103                        if o.timed_out {
3104                            "agent timed out".to_owned()
3105                        } else {
3106                            format!(
3107                                "agent exited with {}",
3108                                crate::notices::exit_status(o.exit_code)
3109                            )
3110                        }
3111                    });
3112                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3113                    (text, o.duration_ms, failed, verified_claim)
3114                }
3115                // Left un-resumed by `resume_undelivered` (a dirty tree
3116                // already rescues the work, or there was no session left to
3117                // resume into) — reported like the ordinary failure it is,
3118                // never as if `o.text` (the CLI's raw error JSON) were an
3119                // answer.
3120                AgentOutcome::Dropped(o) => {
3121                    let why = o
3122                        .dropped
3123                        .as_ref()
3124                        .map(|d| d.why.as_str())
3125                        .unwrap_or("the CLI ended the stream without delivering its answer");
3126                    (
3127                        String::new(),
3128                        o.duration_ms,
3129                        Some(format!("the CLI dropped the stream ({why})")),
3130                        None,
3131                    )
3132                }
3133                AgentOutcome::Quota(o) => {
3134                    self.state.quota.push(QuotaLoss::from_report(
3135                        seat_key,
3136                        "implement",
3137                        o.quota.as_ref(),
3138                    ));
3139                    (
3140                        String::new(),
3141                        o.duration_ms,
3142                        Some("rate limited (quota); produced no change".to_owned()),
3143                        None,
3144                    )
3145                }
3146                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3147                    (String::new(), 0, Some(e), None)
3148                }
3149            };
3150
3151            // Save every completed turn before any fallible Git operation. A failed
3152            // rescue must not erase the reply or prevent collection of other seats.
3153            let c = &mut self.state.candidates[i];
3154            if !exhausted_the_fallback_chain {
3155                c.agent = agent;
3156            }
3157            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3158            c.duration_ms = duration;
3159            c.failed = failed;
3160            c.verified_noop = verified_claim;
3161            self.state
3162                .candidate_preservation
3163                .insert(label, "awaiting persistence".to_owned());
3164        }
3165        self.state.save()?;
3166        self.persist_candidates().await?;
3167
3168        if resource_blocked {
3169            return self.stop_for_resource();
3170        }
3171        if selected_failed {
3172            self.state.status = RunStatus::Blocked;
3173            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3174            self.state.save()?;
3175            bail!(
3176                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3177            );
3178        }
3179        self.after_implement()
3180    }
3181
3182    /// Retry only persistence after a completed implementation turn.
3183    async fn persist_candidates(&mut self) -> Result<()> {
3184        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3185        for label in labels {
3186            let i = self
3187                .state
3188                .candidates
3189                .iter()
3190                .position(|c| c.label == label)
3191                .context("pending candidate is missing")?;
3192            let c = self.state.candidates[i].clone();
3193            let interrupted = self.state.interrupted_implementations.contains(&label);
3194            let result = async {
3195                let wait = crate::repo_lock::WaitPolicy::new(
3196                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3197                    || false,
3198                );
3199                let _coordination =
3200                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3201                // Best effort evidence requires no index writes. A successful snapshot
3202                // never authorizes removal of a tree whose rescue failed.
3203                if let Err(e) = git::snapshot_uncommitted(
3204                    &c.worktree,
3205                    &self
3206                        .state
3207                        .dir()
3208                        .join("artifacts")
3209                        .join(format!("candidate-{label}-recovery")),
3210                )
3211                .await
3212                {
3213                    self.state.event(
3214                        "preserve",
3215                        format!("candidate {label} snapshot incomplete: {e:#}"),
3216                    );
3217                }
3218                let rescued = git::rescue_commit(
3219                    &c.worktree,
3220                    &format!("magi: candidate {label} (uncommitted work)"),
3221                )
3222                .await?;
3223                self.state.note_withheld("implement", &rescued.withheld);
3224                let base = &self.state.base_commit;
3225                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3226                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3227                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3228                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3229                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3230                if c.empty && commits > 0 && !patch.trim().is_empty()
3231                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3232                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3233                        || self.state.tally.is_some())
3234                {
3235                    // Keep the old classification and pending marker so every
3236                    // retry stops too. Retain decisions as recovery evidence;
3237                    // only a supported recovery operation may replace them.
3238                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3239                }
3240                let c = &mut self.state.candidates[i];
3241                c.commits = commits;
3242                c.stat = stat;
3243                c.files = files;
3244                let no_changes = commits == 0 || patch.trim().is_empty();
3245                c.empty = no_changes && !interrupted;
3246                if interrupted && no_changes {
3247                    c.failed = None;
3248                    c.verified_noop = None;
3249                }
3250                if !c.empty {
3251                    c.failed = None;
3252                    c.verified_noop = None;
3253                }
3254                self.state.event(
3255                    "preserve",
3256                    format!("candidate {label}: {files} files, {commits} commits"),
3257                );
3258                Ok::<_, anyhow::Error>(())
3259            }
3260            .await;
3261            match result {
3262                Ok(()) => {
3263                    self.state.candidate_preservation.remove(&label);
3264                    if !interrupted || self.state.candidates[i].files > 0 {
3265                        self.state.interrupted_implementations.remove(&label);
3266                    }
3267                }
3268                Err(e) => {
3269                    let reason = format!("candidate {label} preservation failed: {e:#}");
3270                    self.state.event("preserve", &reason);
3271                    self.state.candidate_preservation.insert(label, reason);
3272                }
3273            }
3274            if self.state.candidate_preservation.is_empty()
3275                && self.state.status == RunStatus::Blocked
3276                && self.state.candidates.iter().all(|c| c.failed.is_none())
3277            {
3278                // Persist the recovered phase with the cleared pending marker;
3279                // a crash between saves must not turn this into a selected-seat
3280                // failure hold on the next resume.
3281                self.state.status = RunStatus::Implementing;
3282                self.state.event(
3283                    "preserve",
3284                    "candidate persistence recovered; implementation can continue",
3285                );
3286            }
3287            self.state.save()?;
3288        }
3289        if !self.state.candidate_preservation.is_empty() {
3290            self.state.status = RunStatus::Blocked;
3291            self.state.save()?;
3292            bail!(
3293                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3294            );
3295        }
3296        Ok(())
3297    }
3298
3299    /// Ask again, once, for work a CLI did and then failed to hand over.
3300    ///
3301    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3302    /// status with an empty response and a usage report showing output tokens,
3303    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3304    /// seven minutes and 14,267 output tokens that arrived as an empty
3305    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3306    ///
3307    /// Two conditions, and both matter:
3308    ///
3309    /// - **Only when the tree is untouched.** Often the agent has already
3310    ///   written its files and only the closing message was lost; the rescue
3311    ///   commit below picks that up and there is nothing to ask for. Re-asking
3312    ///   then would pay for a second implementation of work already on disk.
3313    /// - **Once.** A CLI that drops one stream can drop the next, and this
3314    ///   node is the most expensive in the graph.
3315    ///
3316    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3317    /// dropped reply still carried its `conversation_id`, so the seat is asked
3318    /// to finish what it was doing rather than sent the whole task again. It
3319    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3320    /// node's - for the same reason a re-ranked judge does: restating finished
3321    /// work is not the work.
3322    ///
3323    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3324    /// same way until it resets, while an abandoned conversation is still
3325    /// there to be picked up.
3326    async fn resume_undelivered(
3327        &mut self,
3328        results: &mut [(usize, SeatState, AgentOutcome)],
3329        sent: &[SeatJob],
3330        prompts: &Prompts,
3331        run_id: &str,
3332    ) {
3333        for (wi, seat, out) in results.iter_mut() {
3334            let Some(dropped) = (match &*out {
3335                AgentOutcome::Dropped(o) => o.dropped.clone(),
3336                _ => None,
3337            }) else {
3338                continue;
3339            };
3340            let Some(job) = sent.get(*wi) else { continue };
3341            // Already on disk? Then only the closing message was lost.
3342            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3343                self.state.event(
3344                    "implement",
3345                    format!(
3346                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3347                         work is in the tree",
3348                        seat.key, dropped.output_tokens, dropped.why
3349                    ),
3350                );
3351                continue;
3352            }
3353            // The re-ask only makes sense as a resume: `resume_after_drop`
3354            // says nothing about the task, trusting the seat to still hold it.
3355            // Without a session to resume — sessions disabled, or this CLI's
3356            // drop shape happened not to carry a session id — that prompt
3357            // would open a brand-new conversation with no context at all,
3358            // which is worse than leaving this as the ordinary failure it
3359            // already is.
3360            if !has_context(&job.spec, seat, job.sessions) {
3361                self.state.event(
3362                    "implement",
3363                    format!(
3364                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3365                         is no session left to resume",
3366                        seat.key, dropped.output_tokens, dropped.why
3367                    ),
3368                );
3369                continue;
3370            }
3371            self.state.event(
3372                "implement",
3373                format!(
3374                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3375                     conversation",
3376                    seat.key, dropped.output_tokens, dropped.why
3377                ),
3378            );
3379            let mut retry = job.clone();
3380            retry.seat = seat.clone();
3381            retry.prompt = prompt::resume_after_drop(&dropped.why);
3382            retry.timeout = retry_budget(job.timeout, true);
3383            retry.stem = format!("{}-resume", job.stem);
3384            let cache = self.state.config.cache_dir();
3385            let ctx = WaveCtx {
3386                carry_seats: false,
3387                pauses: Some((&self.pause, &self.interrupt)),
3388                run: run_id,
3389                node: "implement",
3390                prompts,
3391                cache: cache.as_deref(),
3392                round: None,
3393            };
3394            let (resumed_seat, resumed) =
3395                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3396            *seat = resumed_seat;
3397            *out = resumed;
3398        }
3399    }
3400
3401    /// Fall an implement seat through to the next untried agent in the
3402    /// implementer roster when it lost to quota — or, since the handover was
3403    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3404    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3405    /// quota path itself being unchanged — instead of leaving the
3406    /// seat's loss final the moment one agent's account runs dry.
3407    ///
3408    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3409    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3410    /// truncates to the single slot rotation picked, so a solo task whose one
3411    /// implementer hits quota mid-run used to have nothing else to try. This
3412    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3413    /// unrotated roster — which is the only place the *other* candidates in
3414    /// the machine's roster still exist once `implementers` has been cut down
3415    /// to size.
3416    ///
3417    /// Walks forward from just past the seat's own original position in the
3418    /// roster, never wrapping back to the front: a later candidate slot (say
3419    /// `beta`, the roster's second entry) must fall through to the *next*
3420    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3421    /// almost certainly a different candidate's own agent already — and once
3422    /// the roster's tail is exhausted there is nothing left to fall through
3423    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3424    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3425    /// let this retry that id forever. The loop keeps falling through until
3426    /// an attempt lands something other than `Quota` or the roster's tail
3427    /// runs out of untried ids, at which point the seat is left exactly as
3428    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3429    /// `QuotaLoss` recorded, the candidate failed/empty.
3430    ///
3431    /// `sent` is taken mutably and updated with the fallback agent's spec:
3432    /// `resume_unconfirmed_commands`, which runs after this and also reads
3433    /// `sent`, must see whichever agent actually ended up answering the seat
3434    /// — reading the stale, original spec there would check session
3435    /// eligibility against the wrong CLI and could hand a fallback agent's
3436    /// session id to the agent that just lost the seat to quota.
3437    ///
3438    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3439    /// — `self.seat` only reuses state when the agent id is unchanged, so
3440    /// handing it a different id already gets this for free. Reusing the old
3441    /// seat would resume a different CLI's session as if it were a
3442    /// continuation of this one.
3443    ///
3444    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3445    /// a quota loss cuts an agent off mid-turn, so anything already in the
3446    /// tree is unfinished work, not a completed candidate a re-ask would pay
3447    /// for twice. A dirty tree is rescued into a commit first (the same
3448    /// neutral-identity rescue `implement`'s own outcome loop gives every
3449    /// candidate) so the next agent starts clean.
3450    ///
3451    /// The new agent gets the implementer's full prompt and full
3452    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3453    /// it has no session and no context, and is implementing the task from
3454    /// nothing, unlike a resumed drop which is only restating work already
3455    /// done.
3456    ///
3457    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3458    /// `implement` event, never into `self.state.quota` — that is what
3459    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3460    /// go unspent, and a seat that ultimately recovered on its second or
3461    /// third agent is not the stalled panel that check exists to catch. Only
3462    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3463    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3464    /// outcome loop already has — this helper never pushes to it itself.
3465    async fn resume_seat_handovers(
3466        &mut self,
3467        results: &mut [(usize, SeatState, AgentOutcome)],
3468        sent: &mut [SeatJob],
3469        prompts: &Prompts,
3470        run_id: &str,
3471    ) -> Result<()> {
3472        let instruction = seeded_instruction(&self.state);
3473        let language = self.state.config.graph.language.clone();
3474        let github_language = self.state.config.graph.github_language.clone();
3475        let brief = self
3476            .state
3477            .advice
3478            .as_ref()
3479            .and_then(|a| a.synthesis.as_deref())
3480            .map(str::to_owned);
3481        let attachments = self.state.attachments.clone();
3482        for (wi, seat, out) in results.iter_mut() {
3483            // Who holds the other candidate seats of this wave right now
3484            // (earlier handovers already written back to `sent`).
3485            let others: BTreeSet<String> = sent
3486                .iter()
3487                .enumerate()
3488                .filter(|(j, _)| j != wi)
3489                .map(|(_, j)| j.spec.id.clone())
3490                .collect();
3491            let Some(job) = sent.get_mut(*wi) else {
3492                continue;
3493            };
3494            // Where the seat's own original agent sits in the roster — the
3495            // fallback walk starts just past here, never at the front, so a
3496            // later candidate slot's quota loss does not fall back onto an
3497            // earlier slot's own agent.
3498            let start = self
3499                .roles
3500                .implementer_roster
3501                .iter()
3502                .position(|s| s.id == job.spec.id)
3503                .unwrap_or(0);
3504            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3505            let mut fallback_attempt = 0usize;
3506            let mut prev: Option<FailClass> = None;
3507            while let Some(cur) = FailClass::of(&*out) {
3508                if !should_hand_over(prev.as_ref(), &cur) {
3509                    break;
3510                }
3511                let Some(next) =
3512                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3513                        .cloned()
3514                else {
3515                    break;
3516                };
3517                tried.insert(next.id.clone());
3518                fallback_attempt += 1;
3519
3520                let coordination = self.coordinate(&job.cwd).await?;
3521                if let Ok(r) = git::rescue_commit(
3522                    &job.cwd,
3523                    &format!(
3524                        "magi: candidate {} (uncommitted work before {} fallback)",
3525                        seat.key,
3526                        if cur == FailClass::Quota {
3527                            "quota"
3528                        } else {
3529                            "handover"
3530                        }
3531                    ),
3532                )
3533                .await
3534                {
3535                    self.state.note_withheld("implement", &r.withheld);
3536                }
3537                drop(coordination);
3538
3539                record_handover(
3540                    &mut self.state,
3541                    "implement",
3542                    &seat.key,
3543                    &seat.agent,
3544                    &next.id,
3545                    &cur,
3546                    &fail_reason(&*out),
3547                );
3548                prev = Some(cur.clone());
3549
3550                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3551                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3552                // Kept in sync on `sent` itself, not just the local retry: a
3553                // later helper (`resume_unconfirmed_commands`) reads `sent`
3554                // after this one returns and must see whichever agent is now
3555                // occupying the seat, not the one that just quota'd out —
3556                // otherwise it would judge session/continuation eligibility
3557                // by the wrong CLI and could resend a fallback's session id
3558                // to the agent that lost it the seat in the first place.
3559                job.spec = next.clone();
3560                let mut retry = job.clone();
3561                retry.seat = new_seat;
3562                retry.prompt = prompt::implement_in(
3563                    &instruction,
3564                    &job.cwd.to_string_lossy(),
3565                    &language,
3566                    &github_language,
3567                    brief.as_deref(),
3568                    &attachments,
3569                );
3570                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3571                let cache = self.state.config.cache_dir();
3572                let ctx = WaveCtx {
3573                    carry_seats: false,
3574                    pauses: Some((&self.pause, &self.interrupt)),
3575                    run: run_id,
3576                    node: "implement",
3577                    prompts,
3578                    cache: cache.as_deref(),
3579                    round: None,
3580                };
3581                let (fallback_seat, fallback_out) = run_one(
3582                    retry,
3583                    Arc::clone(&self.sem),
3584                    &ctx,
3585                    &mut self.state,
3586                    fallback_attempt,
3587                )
3588                .await;
3589                *seat = fallback_seat;
3590                *out = fallback_out;
3591            }
3592        }
3593        Ok(())
3594    }
3595
3596    /// Ask an implement seat's own CLI to confirm what it started, once, when
3597    /// its reply reported a command whose completion status it never
3598    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3599    /// that does and does not mean.
3600    ///
3601    /// The completion contract this task asks for, extended to `implement`
3602    /// with the same signal `continue_fix_report` reads for the fixer,
3603    /// rather than a keyword search over the reply or a hard requirement on
3604    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3605    /// a candidate's CLI turn ended cleanly while a test run it had started
3606    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3607    /// named in it at all is untouched by this: `commands` is empty, so
3608    /// there is nothing to be unconfirmed.
3609    ///
3610    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3611    /// this is not about recovering edits that might already be on disk, it
3612    /// is about a result the seat itself never vouched for, which resuming
3613    /// asks for regardless of what the tree already holds. Bounded to one
3614    /// attempt for the same reason `resume_undelivered` is — this is the
3615    /// most expensive node in the graph — and a seat that still cannot
3616    /// confirm on that attempt is left as whatever its (possibly still
3617    /// unconfirmed) reply says; this does not invent a new "failed" reason
3618    /// for a candidate that otherwise produced a real, committed change.
3619    async fn resume_unconfirmed_commands(
3620        &mut self,
3621        results: &mut [(usize, SeatState, AgentOutcome)],
3622        sent: &[SeatJob],
3623        prompts: &Prompts,
3624        run_id: &str,
3625    ) {
3626        for (wi, seat, out) in results.iter_mut() {
3627            let AgentOutcome::Ok(o) = &*out else {
3628                continue;
3629            };
3630            if !has_unconfirmed_command(&o.commands) {
3631                continue;
3632            }
3633            let Some(job) = sent.get(*wi) else { continue };
3634            if !has_context(&job.spec, seat, job.sessions) {
3635                self.state.event(
3636                    "implement",
3637                    format!(
3638                        "{}: the reply named a command whose own CLI never confirmed the exit \
3639                         status of, but there is no session left to resume",
3640                        seat.key
3641                    ),
3642                );
3643                continue;
3644            }
3645            self.state.event(
3646                "implement",
3647                format!(
3648                    "{}: the reply named a command whose own CLI never confirmed the exit \
3649                     status of; resuming the conversation",
3650                    seat.key
3651                ),
3652            );
3653            let mut retry = job.clone();
3654            retry.seat = seat.clone();
3655            retry.prompt = prompt::resume_incomplete(
3656                "a command in your last reply had no confirmed exit status",
3657            );
3658            retry.timeout = retry_budget(job.timeout, true);
3659            retry.stem = format!("{}-confirm", job.stem);
3660            let cache = self.state.config.cache_dir();
3661            let ctx = WaveCtx {
3662                carry_seats: false,
3663                pauses: Some((&self.pause, &self.interrupt)),
3664                run: run_id,
3665                node: "implement",
3666                prompts,
3667                cache: cache.as_deref(),
3668                round: None,
3669            };
3670            let (resumed_seat, resumed) =
3671                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3672            *seat = resumed_seat;
3673            *out = resumed;
3674        }
3675    }
3676
3677    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3678    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3679    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3680    /// that motivated this.
3681    ///
3682    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3683    /// own nudge loop already covers for judge/review/vote seats, and not a
3684    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3685    /// implement seats: here the CLI turn genuinely finished while the node's
3686    /// own work — the fixer's account of what it did — had not. Gated purely
3687    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3688    /// reply, never on any wording in it, so a fixer whose valid, first-try
3689    /// `FixReport` happens to mention having waited on a background test is
3690    /// never resumed — the `Ok(report)` branch at the call site returns
3691    /// before this is ever invoked.
3692    ///
3693    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3694    /// attempt ([`retry_budget`]), nothing attempted once the session is
3695    /// gone, and a quota hit ends the loop immediately rather than retrying a
3696    /// rate limit that fails the same way again.
3697    async fn continue_fix_report(
3698        &mut self,
3699        mut seat: SeatState,
3700        parse_err: String,
3701        job: &SeatJob,
3702        prompts: &Prompts,
3703        run_id: &str,
3704        round: usize,
3705    ) -> (
3706        SeatState,
3707        Option<FixReport>,
3708        Option<String>,
3709        ContinuationRecord,
3710    ) {
3711        let mut last_err = parse_err;
3712        let mut cumulative_wait_ms = 0u64;
3713        let mut attempts = 0usize;
3714        loop {
3715            if !has_context(&job.spec, &seat, job.sessions) {
3716                self.state.event(
3717                    "fix",
3718                    format!(
3719                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3720                         session left to resume into"
3721                    ),
3722                );
3723                let outcome = if attempts == 0 {
3724                    ContinuationOutcome::NoSession
3725                } else {
3726                    ContinuationOutcome::Exhausted
3727                };
3728                return (
3729                    seat,
3730                    None,
3731                    Some(format!("unparsable fix report: {last_err}")),
3732                    ContinuationRecord {
3733                        attempts,
3734                        cumulative_wait_ms,
3735                        outcome,
3736                    },
3737                );
3738            }
3739            if attempts >= MAX_FIX_CONTINUATIONS {
3740                self.state.event(
3741                    "fix",
3742                    format!(
3743                        "round {round}: fixer's reply still had no adoption report after \
3744                         {attempts} continuation(s) ({last_err}); giving up"
3745                    ),
3746                );
3747                return (
3748                    seat,
3749                    None,
3750                    Some(format!(
3751                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3752                    )),
3753                    ContinuationRecord {
3754                        attempts,
3755                        cumulative_wait_ms,
3756                        outcome: ContinuationOutcome::Exhausted,
3757                    },
3758                );
3759            }
3760            attempts += 1;
3761            self.state.event(
3762                "fix",
3763                format!(
3764                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3765                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3766                ),
3767            );
3768            let mut retry = job.clone();
3769            retry.seat = seat.clone();
3770            retry.prompt = prompt::resume_incomplete(&last_err);
3771            retry.timeout = retry_budget(job.timeout, true);
3772            retry.stem = format!("{}-continue{attempts}", job.stem);
3773            let cache = self.state.config.cache_dir();
3774            let ctx = WaveCtx {
3775                carry_seats: false,
3776                pauses: Some((&self.pause, &self.interrupt)),
3777                run: run_id,
3778                node: "fix",
3779                prompts,
3780                cache: cache.as_deref(),
3781                round: Some(round),
3782            };
3783            let (resumed_seat, resumed_out) = run_one(
3784                retry,
3785                Arc::clone(&self.sem),
3786                &ctx,
3787                &mut self.state,
3788                attempts,
3789            )
3790            .await;
3791            seat = resumed_seat;
3792            match resumed_out {
3793                AgentOutcome::Ok(o) => {
3794                    cumulative_wait_ms += o.duration_ms;
3795                    match verdict::extract_json::<FixReport>(&o.text) {
3796                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3797                            self.state.event(
3798                                "fix",
3799                                format!(
3800                                    "round {round}: fixer's adoption report recovered after \
3801                                     {attempts} continuation(s)"
3802                                ),
3803                            );
3804                            return (
3805                                seat,
3806                                Some(report),
3807                                None,
3808                                ContinuationRecord {
3809                                    attempts,
3810                                    cumulative_wait_ms,
3811                                    outcome: ContinuationOutcome::Resumed,
3812                                },
3813                            );
3814                        }
3815                        // The report parsed, but this same reply's own
3816                        // CommandEvidence — the identical record `state.jobs`
3817                        // renders — names a command whose CLI never
3818                        // confirmed an exit status. Read together, that is
3819                        // not a resolved answer: keep nudging rather than
3820                        // accept a report standing next to a command the
3821                        // seat's own CLI cannot vouch for.
3822                        Ok(_) => {
3823                            last_err = "the reply parsed, but it reported a command whose own CLI \
3824                                 never confirmed an exit status"
3825                                .to_owned();
3826                        }
3827                        Err(e) => last_err = e.to_string(),
3828                    }
3829                }
3830                AgentOutcome::Quota(o) => {
3831                    cumulative_wait_ms += o.duration_ms;
3832                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3833                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3834                    self.state.quota.push(loss);
3835                    self.state.event(
3836                        "fix",
3837                        format!(
3838                            "round {round}: continuation rate limited (quota); not retrying now"
3839                        ),
3840                    );
3841                    return (
3842                        seat,
3843                        None,
3844                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3845                        ContinuationRecord {
3846                            attempts,
3847                            cumulative_wait_ms,
3848                            outcome: ContinuationOutcome::QuotaLost,
3849                        },
3850                    );
3851                }
3852                AgentOutcome::Dropped(o) => {
3853                    cumulative_wait_ms += o.duration_ms;
3854                    let why = o
3855                        .dropped
3856                        .as_ref()
3857                        .map(|d| d.why.as_str())
3858                        .unwrap_or("the CLI ended the stream without delivering its answer");
3859                    last_err = format!("the CLI dropped the stream ({why})");
3860                }
3861                AgentOutcome::Failed(e) => last_err = e,
3862                // The continuation was never started; the report stays lost
3863                // and the caller records that, without any handover.
3864                AgentOutcome::ResourceBlocked(e) => {
3865                    return (
3866                        seat,
3867                        None,
3868                        Some(format!("the continuation was not started: {e}")),
3869                        ContinuationRecord {
3870                            attempts: attempts.saturating_sub(1),
3871                            cumulative_wait_ms,
3872                            outcome: ContinuationOutcome::NoSession,
3873                        },
3874                    );
3875                }
3876            }
3877        }
3878    }
3879
3880    fn after_implement(&mut self) -> Result<()> {
3881        if !self.state.candidate_preservation.is_empty() {
3882            bail!("candidate preservation incomplete; judging is unsafe");
3883        }
3884        // Scan every candidate patch once the set is complete.
3885        if self.state.leaks.is_empty() {
3886            let cfg = self.state.config.blind.clone();
3887            let mut leaks = Vec::new();
3888            for c in &self.state.candidates {
3889                let Some(patch) =
3890                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3891                else {
3892                    continue;
3893                };
3894                leaks.extend(blind::scan(
3895                    &format!("candidate {} patch", c.label),
3896                    &patch,
3897                    &cfg.vendor_tokens,
3898                ));
3899            }
3900            if !leaks.is_empty() {
3901                let summary = leaks
3902                    .iter()
3903                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3904                    .collect::<Vec<_>>()
3905                    .join(", ");
3906                match cfg.on_leak {
3907                    LeakPolicy::Fail => {
3908                        self.state.status = RunStatus::Failed;
3909                        self.state
3910                            .event("blind", format!("vendor text in a patch: {summary}"));
3911                        self.state.leaks = leaks;
3912                        self.state.save()?;
3913                        self.settle_questions();
3914                        bail!(
3915                            "blind.on_leak = \"fail\" and vendor text reached a \
3916                             judged patch: {summary}"
3917                        );
3918                    }
3919                    LeakPolicy::Redact => self.state.event(
3920                        "blind",
3921                        format!("redacting vendor text for judging: {summary}"),
3922                    ),
3923                    LeakPolicy::Warn => self.state.event(
3924                        "blind",
3925                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3926                    ),
3927                }
3928                self.state.leaks = leaks;
3929            }
3930        }
3931
3932        if self.state.viable().is_empty() {
3933            if self.state.all_candidates_verified_noop() {
3934                // Every candidate agreed, with evidence the adoption guard
3935                // accepted, that nothing belongs in this worktree. That is
3936                // not the same fact as a candidate that simply failed to
3937                // write anything, and settling it as an ordinary `Failed`
3938                // (see `SCHEMA`'s doc for schema 10) is what let two of
3939                // task 391f's attempts burn a retry each re-discovering the
3940                // same already-landed fix. Terminal either way, so `judge`
3941                // must never run over an empty candidate set — unlike the
3942                // `Failed` branch below this returns `Ok`, not an error:
3943                // nothing here failed.
3944                self.state.status = RunStatus::VerifiedNoop;
3945                self.state.save()?;
3946                self.settle_questions();
3947                return Ok(());
3948            }
3949            self.state.status = RunStatus::Failed;
3950            self.state.save()?;
3951            self.settle_questions();
3952            bail!("no candidate produced a change; nothing to judge");
3953        }
3954        self.state.status = RunStatus::Judging;
3955        self.state.save()?;
3956        Ok(())
3957    }
3958
3959    // --------------------------------------------------------------- judge
3960
3961    async fn judge(&mut self) -> Result<()> {
3962        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3963        // agent files with `magi task add` name the run that paid for it. The
3964        // prompt overlay is cloned alongside it because the waves borrow it
3965        // while `self` is mutably borrowed by the node's own bookkeeping.
3966        let run_id = self.state.id.clone();
3967        let prompts = self.state.config.prompts.clone();
3968        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3969            return Ok(());
3970        }
3971        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3972        if viable.len() == 1 {
3973            // Recorded so this is a one-time event: `judgements` stays empty
3974            // either way, which without this flag is indistinguishable from
3975            // "not yet judged" on the next reentry — and status is left
3976            // untouched, so a later node's conclusion (e.g. `Blocked` after
3977            // the review budget ran out) survives a resume instead of being
3978            // clobbered back to `Judging` by this node running again.
3979            self.state.judge_skipped = true;
3980            self.state.event(
3981                "judge",
3982                format!(
3983                    "only candidate {} produced a change; judging skipped",
3984                    viable[0].label
3985                ),
3986            );
3987            self.state.save()?;
3988            return Ok(());
3989        }
3990        self.state.status = RunStatus::Judging;
3991
3992        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3993        let language = self.state.config.graph.language.clone();
3994        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3995        let sessions = self.state.config.graph.sessions;
3996        let artifacts = agent::artifacts_dir(&self.state.dir());
3997        let root = self.state.worktree_root();
3998        let base_short = short(&self.state.base_commit);
3999
4000        let mut jobs = Vec::new();
4001        let mut orders = Vec::new();
4002        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4003            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4004            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4005            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4006            let seat_key = format!("judge-{}", j + 1);
4007            let seat = self.seat(&seat_key, &spec.id);
4008            jobs.push(SeatJob {
4009                prompt: prompt::judge(
4010                    &self.state.instruction,
4011                    &views,
4012                    self.roles.judges.len(),
4013                    &base_short,
4014                    &language,
4015                ),
4016                spec,
4017                seat,
4018                cwd: root.join(format!("judge-{}", j + 1)),
4019                timeout,
4020                allow_write: false,
4021                sessions,
4022                artifacts: artifacts.clone(),
4023                stem: format!("judge-{}", j + 1),
4024                handover: None,
4025            });
4026        }
4027
4028        self.state.event(
4029            "judge",
4030            format!(
4031                "{} judges ranking {} candidates blind",
4032                jobs.len(),
4033                viable.len()
4034            ),
4035        );
4036        let labels_for_check = labels.clone();
4037        let mut quota_losses = Vec::new();
4038        let cache = self.state.config.cache_dir();
4039        let ctx = WaveCtx {
4040            carry_seats: false,
4041            pauses: Some((&self.pause, &self.interrupt)),
4042            run: &run_id,
4043            node: "judge",
4044            prompts: &prompts,
4045            cache: cache.as_deref(),
4046            round: None,
4047        };
4048        let results = ask_json_wave::<Ranking>(
4049            jobs,
4050            Arc::clone(&self.sem),
4051            self.state.config.graph.retries,
4052            &self.roles.judge_roster,
4053            &ctx,
4054            &mut quota_losses,
4055            &mut self.state,
4056            &move |r: &Ranking| r.validate(&labels_for_check),
4057        )
4058        .await;
4059        if self.state.resource_block.is_some() {
4060            return self.stop_for_resource();
4061        }
4062        self.state.quota.extend(quota_losses);
4063
4064        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4065            let agent_id = seat.agent.clone();
4066            self.state.seats.insert(seat.key.clone(), seat);
4067            let mut record = Judgement {
4068                judge: j + 1,
4069                seat: format!("judge-{}", j + 1),
4070                agent: agent_id,
4071                ranking: Vec::new(),
4072                reasons: BTreeMap::new(),
4073                confidence: None,
4074                strengths: BTreeMap::new(),
4075                order: orders[j].clone(),
4076                failed: None,
4077                duration_ms: 0,
4078            };
4079            match res {
4080                Ok((ranking, out)) => {
4081                    record.ranking = ranking.normalized();
4082                    record.reasons = ranking.reasons;
4083                    record.strengths = ranking.strengths;
4084                    record.confidence = ranking.confidence;
4085                    record.duration_ms = out.duration_ms;
4086                    self.state.event(
4087                        "judge",
4088                        format!(
4089                            "judge {} ranked {}",
4090                            j + 1,
4091                            record.ranking.iter().collect::<String>()
4092                        ),
4093                    );
4094                }
4095                Err(e) => {
4096                    record.failed = Some(e.to_string());
4097                    self.state
4098                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4099                }
4100            }
4101            self.state.judgements.push(record);
4102            self.state.save()?;
4103        }
4104        Ok(())
4105    }
4106
4107    // ---------------------------------------------------------- deliberate
4108
4109    async fn deliberate(&mut self) -> Result<()> {
4110        if self.hold_base_sync().await? {
4111            return Ok(());
4112        }
4113        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4114        // agent files with `magi task add` name the run that paid for it. The
4115        // prompt overlay is cloned alongside it because the waves borrow it
4116        // while `self` is mutably borrowed by the node's own bookkeeping.
4117        let run_id = self.state.id.clone();
4118        let prompts = self.state.config.prompts.clone();
4119        let rounds = self.state.config.graph.deliberate_rounds;
4120        // Resume unfinished rounds after an outage, but never reopen a tally.
4121        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4122            return Ok(());
4123        }
4124        let tops: Vec<char> = self
4125            .state
4126            .judgements
4127            .iter()
4128            .filter_map(|j| j.ranking.first().copied())
4129            .collect();
4130        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4131            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4132                self.state.event(
4133                    "deliberate",
4134                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4135                );
4136            }
4137            self.state.status = RunStatus::Voting;
4138            self.state.save()?;
4139            return Ok(());
4140        }
4141
4142        self.state.status = RunStatus::Deliberating;
4143        self.state.event(
4144            "deliberate",
4145            format!(
4146                "split: first choices were {} — opening {rounds} round(s)",
4147                tops.iter().collect::<String>()
4148            ),
4149        );
4150
4151        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4152        let language = self.state.config.graph.language.clone();
4153        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4154        let sessions = self.state.config.graph.sessions;
4155        let artifacts = agent::artifacts_dir(&self.state.dir());
4156        let root = self.state.worktree_root();
4157        let base_short = short(&self.state.base_commit);
4158
4159        // Judges argue in sequence so that a turn can answer the one before it;
4160        // that is the difference between deliberation and three parallel
4161        // monologues.
4162        for round in self.state.deliberation.len() + 1..=rounds {
4163            let mut turns: Vec<DeliberationTurn> = Vec::new();
4164            let mut round_losses = Vec::new();
4165            let mut asked = 0;
4166            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4167                if self.state.judgements[j].failed.is_some() {
4168                    continue;
4169                }
4170                let seat_key = format!("judge-{}", j + 1);
4171                let spec = self.occupant(&seat_key, spec);
4172                let mut seat = self.seat(&seat_key, &spec.id);
4173                let transcript = self.transcript(&turns, j);
4174                let build = |context: Option<&str>| {
4175                    prompt::deliberate(
4176                        &self.state.instruction,
4177                        context,
4178                        &transcript,
4179                        round,
4180                        rounds,
4181                        &language,
4182                    )
4183                };
4184                let block = self.candidate_block(&viable, &base_short);
4185                let full = build(Some(&block));
4186                let text = if has_context(&spec, &seat, sessions) {
4187                    build(None)
4188                } else {
4189                    full.clone()
4190                };
4191                let job = SeatJob {
4192                    spec,
4193                    seat: seat.clone(),
4194                    prompt: text,
4195                    cwd: root.join(format!("judge-{}", j + 1)),
4196                    timeout,
4197                    allow_write: false,
4198                    sessions,
4199                    artifacts: artifacts.clone(),
4200                    stem: format!("delib-{round}-judge-{}", j + 1),
4201                    handover: Some(full),
4202                };
4203                let cache = self.state.config.cache_dir();
4204                let ctx = WaveCtx {
4205                    carry_seats: false,
4206                    pauses: Some((&self.pause, &self.interrupt)),
4207                    run: &run_id,
4208                    node: "deliberate",
4209                    prompts: &prompts,
4210                    cache: cache.as_deref(),
4211                    round: None,
4212                };
4213                // A turn is never nudged (`retries` 0): a failed seat is
4214                // handed to the next roster agent, which gets the full
4215                // context. An empty answer is a turn, not a failure.
4216                let mut losses = Vec::new();
4217                asked += 1;
4218                let mut results = ask_wave_with::<String>(
4219                    vec![job],
4220                    Arc::clone(&self.sem),
4221                    0,
4222                    &self.roles.judge_roster,
4223                    &ctx,
4224                    &mut losses,
4225                    &mut self.state,
4226                    &|text: &str| {
4227                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4228                    },
4229                )
4230                .await;
4231                if self.state.resource_block.is_some() {
4232                    return self.stop_for_resource();
4233                }
4234                round_losses.extend(losses.iter().cloned());
4235                self.state.quota.extend(losses);
4236                let (updated, res, _) = results.pop().expect("one job in, one result out");
4237                seat = updated;
4238                let agent_id = seat.agent.clone();
4239                self.state.seats.insert(seat.key.clone(), seat);
4240                let body = match res {
4241                    Ok((body, _)) => body,
4242                    // Skip the seat; a CLI's raw error JSON is never read as
4243                    // this judge's position.
4244                    Err(e) => {
4245                        self.state
4246                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4247                        continue;
4248                    }
4249                };
4250                let tentative = verdict::extract_json::<Position>(&body)
4251                    .ok()
4252                    .and_then(|p| p.tentative)
4253                    .and_then(|s| s.trim().chars().next())
4254                    .map(|c| c.to_ascii_uppercase());
4255                self.state.event(
4256                    "deliberate",
4257                    format!(
4258                        "round {round}: judge {} now favours {}",
4259                        j + 1,
4260                        tentative.map_or("—".to_owned(), |c| c.to_string())
4261                    ),
4262                );
4263                turns.push(DeliberationTurn {
4264                    judge: j + 1,
4265                    agent: agent_id,
4266                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4267                    tentative,
4268                });
4269            }
4270            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4271                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4272                return self.stop_for_resource();
4273            }
4274            self.state
4275                .deliberation
4276                .push(DeliberationRound { round, turns });
4277            self.state.save()?;
4278        }
4279
4280        self.state.status = RunStatus::Voting;
4281        self.state.save()?;
4282        Ok(())
4283    }
4284
4285    // ---------------------------------------------------------------- vote
4286
4287    async fn vote(&mut self) -> Result<()> {
4288        if self.hold_base_sync().await? {
4289            return Ok(());
4290        }
4291        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4292        // agent files with `magi task add` name the run that paid for it. The
4293        // prompt overlay is cloned alongside it because the waves borrow it
4294        // while `self` is mutably borrowed by the node's own bookkeeping.
4295        let run_id = self.state.id.clone();
4296        let prompts = self.state.config.prompts.clone();
4297        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4298            return Ok(());
4299        }
4300        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4301        if viable.len() == 1 {
4302            return Ok(());
4303        }
4304        self.state.status = RunStatus::Voting;
4305
4306        let language = self.state.config.graph.language.clone();
4307        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4308        let sessions = self.state.config.graph.sessions;
4309        let artifacts = agent::artifacts_dir(&self.state.dir());
4310        let root = self.state.worktree_root();
4311        let base_short = short(&self.state.base_commit);
4312        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4313
4314        let mut jobs = Vec::new();
4315        let mut seats_at = Vec::new();
4316        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4317            if self
4318                .state
4319                .judgements
4320                .get(j)
4321                .is_some_and(|r| r.failed.is_some())
4322            {
4323                continue;
4324            }
4325            let seat_key = format!("judge-{}", j + 1);
4326            let spec = self.occupant(&seat_key, spec);
4327            let seat = self.seat(&seat_key, &spec.id);
4328            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4329            let text = if has_context(&spec, &seat, sessions) {
4330                prompt::final_vote(&viable, &language)
4331            } else {
4332                full.clone()
4333            };
4334            jobs.push(SeatJob {
4335                spec,
4336                seat,
4337                prompt: text,
4338                cwd: root.join(format!("judge-{}", j + 1)),
4339                timeout,
4340                allow_write: false,
4341                sessions,
4342                artifacts: artifacts.clone(),
4343                stem: format!("vote-judge-{}", j + 1),
4344                handover: Some(full),
4345            });
4346            seats_at.push(j);
4347        }
4348
4349        self.state.event(
4350            "vote",
4351            format!(
4352                "collecting {} final votes one by one, privately",
4353                jobs.len()
4354            ),
4355        );
4356        let allowed = viable.clone();
4357        let mut quota_losses = Vec::new();
4358        let cache = self.state.config.cache_dir();
4359        let ctx = WaveCtx {
4360            carry_seats: false,
4361            pauses: Some((&self.pause, &self.interrupt)),
4362            run: &run_id,
4363            node: "vote",
4364            prompts: &prompts,
4365            cache: cache.as_deref(),
4366            round: None,
4367        };
4368        let results = ask_json_wave::<FinalVote>(
4369            jobs,
4370            Arc::clone(&self.sem),
4371            self.state.config.graph.retries,
4372            &self.roles.judge_roster,
4373            &ctx,
4374            &mut quota_losses,
4375            &mut self.state,
4376            &move |v: &FinalVote| match v.label() {
4377                Some(c) if allowed.contains(&c) => Ok(()),
4378                other => bail!("vote {other:?} is not one of {allowed:?}"),
4379            },
4380        )
4381        .await;
4382        if self.state.resource_block.is_some() {
4383            return self.stop_for_resource();
4384        }
4385        self.state.quota.extend(quota_losses);
4386
4387        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4388            let agent_id = seat.agent.clone();
4389            self.state.seats.insert(seat.key.clone(), seat);
4390            let initial = self
4391                .state
4392                .judgements
4393                .get(j)
4394                .and_then(|r| r.ranking.first().copied());
4395            let mut record = VoteRecord {
4396                judge: j + 1,
4397                agent: agent_id,
4398                vote: None,
4399                reason: String::new(),
4400                changed: false,
4401            };
4402            match res {
4403                Ok((v, _)) => {
4404                    record.vote = v.label();
4405                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4406                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4407                    self.state.event(
4408                        "vote",
4409                        format!(
4410                            "judge {} voted {}{}",
4411                            j + 1,
4412                            record.vote.unwrap_or('?'),
4413                            if record.changed { " (changed)" } else { "" }
4414                        ),
4415                    );
4416                }
4417                Err(e) => {
4418                    self.state
4419                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4420                }
4421            }
4422            self.state.votes.push(record);
4423            self.state.save()?;
4424        }
4425        Ok(())
4426    }
4427
4428    // --------------------------------------------------------------- tally
4429
4430    fn tally(&mut self) -> Result<()> {
4431        if self.state.tally.is_some() {
4432            return Ok(());
4433        }
4434        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4435        let tops: Vec<char> = self
4436            .state
4437            .judgements
4438            .iter()
4439            .filter_map(|j| j.ranking.first().copied())
4440            .collect();
4441        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4442
4443        // A judge whose private vote failed still counted once, in the initial
4444        // ranking; using it beats discarding a whole seat.
4445        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4446        let mut cast: Vec<char> = Vec::new();
4447        for (i, j) in self.state.judgements.iter().enumerate() {
4448            let vote = self
4449                .state
4450                .votes
4451                .iter()
4452                .find(|v| v.judge == i + 1)
4453                .and_then(|v| v.vote)
4454                .or_else(|| j.ranking.first().copied());
4455            if let Some(v) = vote {
4456                *first_choice.entry(v).or_insert(0) += 1;
4457                cast.push(v);
4458            }
4459        }
4460
4461        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4462        for j in &self.state.judgements {
4463            let n = j.ranking.len();
4464            for (pos, label) in j.ranking.iter().enumerate() {
4465                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4466            }
4467        }
4468
4469        let best = first_choice.values().copied().max().unwrap_or(0);
4470        let mut leaders: Vec<char> = first_choice
4471            .iter()
4472            .filter(|(_, v)| **v == best)
4473            .map(|(k, _)| *k)
4474            .collect();
4475        let mut tie_break = None;
4476        if leaders.len() > 1 {
4477            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4478            let borda_leaders: Vec<char> = leaders
4479                .iter()
4480                .copied()
4481                .filter(|l| borda[l] == top_borda)
4482                .collect();
4483            tie_break = Some(if borda_leaders.len() == 1 {
4484                format!(
4485                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4486                    leaders.len()
4487                )
4488            } else {
4489                format!(
4490                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4491                    leaders.len()
4492                )
4493            });
4494            leaders = borda_leaders;
4495            leaders.sort_unstable();
4496        }
4497        let winner = *leaders
4498            .first()
4499            .or(viable.first())
4500            .context("no candidate to declare a winner from")?;
4501
4502        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4503        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4504        let deliberated = !self.state.deliberation.is_empty();
4505
4506        // Whose verdict is this? A rate-limited seat is absent even if it
4507        // ranked before the limit hit, so presence is measured against the
4508        // recorded losses, not just "did a ranking ever appear".
4509        let quota_seats: std::collections::BTreeSet<&str> =
4510            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4511        let mut present = 0usize;
4512        for (i, j) in self.state.judgements.iter().enumerate() {
4513            if quota_seats.contains(j.seat.as_str()) {
4514                continue;
4515            }
4516            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4517            let voted = self
4518                .state
4519                .votes
4520                .iter()
4521                .any(|v| v.judge == i + 1 && v.vote.is_some());
4522            if ranked || voted {
4523                present += 1;
4524            }
4525        }
4526        // Strict majority of the configured panel. A bare majority is real
4527        // signal we can act on, while a minority verdict must never stand in
4528        // for a healthy one. A one-candidate run needs no panel at all, and
4529        // `judges` stays `0` rather than the roster size a panel that never
4530        // sat would otherwise be credited with.
4531        let needs_quorum = viable.len() > 1;
4532        let judges_total = if needs_quorum {
4533            self.roles.judges.len()
4534        } else {
4535            0
4536        };
4537        let quorum = if needs_quorum {
4538            judges_total / 2 + 1
4539        } else {
4540            0
4541        };
4542        let met_quorum = !needs_quorum || present >= quorum;
4543        let uncontested = (!needs_quorum).then(|| {
4544            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4545        });
4546
4547        self.state.event(
4548            "tally",
4549            match &uncontested {
4550                Some(reason) => format!("winner {winner} — {reason}"),
4551                None => format!(
4552                    "winner {winner} — votes {} | initial {} | {} changed | \
4553                     {present}/{judges_total} judges{}",
4554                    first_choice
4555                        .iter()
4556                        .map(|(k, v)| format!("{k}:{v}"))
4557                        .collect::<Vec<_>>()
4558                        .join(" "),
4559                    if unanimous_initial {
4560                        "unanimous"
4561                    } else {
4562                        "split"
4563                    },
4564                    changed_votes,
4565                    if met_quorum {
4566                        String::new()
4567                    } else {
4568                        format!(" — below quorum ({quorum} required)")
4569                    },
4570                ),
4571            },
4572        );
4573        if !met_quorum {
4574            self.state.event(
4575                "stall",
4576                format!(
4577                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4578                     the run stops here, resumable"
4579                ),
4580            );
4581        }
4582        self.state.tally = Some(Tally {
4583            first_choice,
4584            borda,
4585            winner,
4586            rankings: tops.len(),
4587            unanimous_initial,
4588            deliberated,
4589            changed_votes,
4590            unanimous_final,
4591            tie_break,
4592            judges: judges_total,
4593            present,
4594            quorum,
4595            met_quorum,
4596            uncontested,
4597        });
4598        self.state.status = if met_quorum {
4599            RunStatus::Reviewing
4600        } else {
4601            RunStatus::Stalled
4602        };
4603        self.state.save()?;
4604        Ok(())
4605    }
4606
4607    // ------------------------------------------------------------- recover
4608
4609    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4610    /// resumed toward completion once the transient cause clears.
4611    ///
4612    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4613    /// it toward the quorum, which is exactly the set of seats whose absence
4614    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4615    /// not depend on which node happened to hit the limit), or an ordinary
4616    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4617    /// seat is never disturbed.
4618    ///
4619    /// A seat that now answers with a usable ranking is "recovered": its
4620    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4621    /// `tally` counts it present again), and its vote re-collected. A seat that
4622    /// still fails keeps its loss and stays absent.
4623    ///
4624    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4625    /// to review/gate/merge), `false` when it is still below quorum (the run
4626    /// stays `Stalled`, still resumable for a later retry).
4627    #[allow(clippy::too_many_lines)]
4628    async fn recover_stall(&mut self) -> Result<bool> {
4629        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4630        // agent files with `magi task add` name the run that paid for it. The
4631        // prompt overlay is cloned alongside it because the waves borrow it
4632        // while `self` is mutably borrowed by the node's own bookkeeping.
4633        let run_id = self.state.id.clone();
4634        let prompts = self.state.config.prompts.clone();
4635        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4636        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4637        // is what keeps a non-quota collapse — or a quota loss recorded at the
4638        // deliberate node — from being a permanent dead-end on `--resume`.
4639        let quota_seats: BTreeSet<&str> =
4640            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4641        let absent: Vec<String> = self
4642            .state
4643            .judgements
4644            .iter()
4645            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4646            .map(|j| j.seat.clone())
4647            .collect();
4648        if absent.is_empty() {
4649            return Ok(false);
4650        }
4651        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4652        if viable.len() <= 1 {
4653            return Ok(false);
4654        }
4655        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4656        let language = self.state.config.graph.language.clone();
4657        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4658        let sessions = self.state.config.graph.sessions;
4659        let artifacts = agent::artifacts_dir(&self.state.dir());
4660        let root = self.state.worktree_root();
4661        let base_short = short(&self.state.base_commit);
4662        let candidates: Vec<Candidate> = viable.clone();
4663
4664        // Map each absent seat key to its 0-based position in `roles.judges`.
4665        let mut positions: Vec<usize> = absent
4666            .iter()
4667            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4668            .collect();
4669        if positions.is_empty() {
4670            return Ok(false);
4671        }
4672        positions.sort_unstable();
4673        positions.dedup();
4674
4675        // Re-rank the lost seats, one blind prompt each.
4676        let mut judge_jobs = Vec::new();
4677        for &j in &positions {
4678            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4679            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4680            let seat_key = format!("judge-{}", j + 1);
4681            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4682            let seat = self.seat(&seat_key, &spec.id);
4683            judge_jobs.push(SeatJob {
4684                spec,
4685                seat,
4686                prompt: prompt::judge(
4687                    &self.state.instruction,
4688                    &views,
4689                    self.roles.judges.len(),
4690                    &base_short,
4691                    &language,
4692                ),
4693                cwd: root.join(seat_key),
4694                timeout,
4695                allow_write: false,
4696                sessions,
4697                artifacts: artifacts.clone(),
4698                stem: format!("judge-{}-recover", j + 1),
4699                handover: None,
4700            });
4701        }
4702
4703        let labels_for_check = labels.clone();
4704        let mut judge_losses = Vec::new();
4705        let retries = self.state.config.graph.retries;
4706        let cache = self.state.config.cache_dir();
4707        let ctx = WaveCtx {
4708            carry_seats: false,
4709            pauses: Some((&self.pause, &self.interrupt)),
4710            run: &run_id,
4711            node: "judge",
4712            prompts: &prompts,
4713            cache: cache.as_deref(),
4714            round: None,
4715        };
4716        let results = ask_json_wave::<Ranking>(
4717            judge_jobs,
4718            Arc::clone(&self.sem),
4719            retries,
4720            &self.roles.judge_roster,
4721            &ctx,
4722            &mut judge_losses,
4723            &mut self.state,
4724            &move |r: &Ranking| r.validate(&labels_for_check),
4725        )
4726        .await;
4727        if self.state.resource_block.is_some() {
4728            self.stop_for_resource()?;
4729            return Ok(false);
4730        }
4731
4732        // Refresh the judgement of every seat that ranked again.
4733        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4734        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4735            let agent_id = seat.agent.clone();
4736            self.state.seats.insert(seat.key.clone(), seat);
4737            let record = &mut self.state.judgements[j];
4738            match res {
4739                Ok((ranking, out)) => {
4740                    record.agent = agent_id;
4741                    record.ranking = ranking.normalized();
4742                    record.reasons = ranking.reasons;
4743                    record.strengths = ranking.strengths;
4744                    record.confidence = ranking.confidence;
4745                    record.failed = None;
4746                    record.duration_ms = out.duration_ms;
4747                    recovered.insert(j);
4748                    self.state.event(
4749                        "recover",
4750                        format!("judge {} ranked again after the limit", j + 1),
4751                    );
4752                }
4753                Err(e) => {
4754                    self.state
4755                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4756                }
4757            }
4758        }
4759
4760        // Re-ask the votes of the seats that recovered a ranking.
4761        let mut vote_jobs = Vec::new();
4762        let mut vote_pos: Vec<usize> = Vec::new();
4763        for &j in &recovered {
4764            let seat_key = format!("judge-{}", j + 1);
4765            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4766            let seat = self.seat(&seat_key, &spec.id);
4767            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4768            let text = if has_context(&spec, &seat, sessions) {
4769                prompt::final_vote(&labels, &language)
4770            } else {
4771                full.clone()
4772            };
4773            vote_jobs.push(SeatJob {
4774                spec,
4775                seat,
4776                prompt: text,
4777                cwd: root.join(seat_key),
4778                timeout,
4779                allow_write: false,
4780                sessions,
4781                artifacts: artifacts.clone(),
4782                stem: format!("vote-judge-{}-recover", j + 1),
4783                handover: Some(full),
4784            });
4785            vote_pos.push(j);
4786        }
4787        let allowed = labels.clone();
4788        let mut vote_losses = Vec::new();
4789        let vote_retries = self.state.config.graph.retries;
4790        let vote_cache = self.state.config.cache_dir();
4791        let ctx = WaveCtx {
4792            carry_seats: false,
4793            pauses: Some((&self.pause, &self.interrupt)),
4794            run: &run_id,
4795            node: "vote",
4796            prompts: &prompts,
4797            cache: vote_cache.as_deref(),
4798            round: None,
4799        };
4800        let votes = ask_json_wave::<FinalVote>(
4801            vote_jobs,
4802            Arc::clone(&self.sem),
4803            vote_retries,
4804            &self.roles.judge_roster,
4805            &ctx,
4806            &mut vote_losses,
4807            &mut self.state,
4808            &move |v: &FinalVote| match v.label() {
4809                Some(c) if allowed.contains(&c) => Ok(()),
4810                other => bail!("vote {other:?} is not one of {allowed:?}"),
4811            },
4812        )
4813        .await;
4814        if self.state.resource_block.is_some() {
4815            self.stop_for_resource()?;
4816            return Ok(false);
4817        }
4818        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4819            let agent_id = seat.agent.clone();
4820            self.state.seats.insert(seat.key.clone(), seat);
4821            match res {
4822                Ok((v, _)) => {
4823                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4824                        rec.vote = v.label();
4825                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4826                    } else {
4827                        self.state.votes.push(VoteRecord {
4828                            judge: j + 1,
4829                            agent: agent_id,
4830                            vote: v.label(),
4831                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4832                            changed: false,
4833                        });
4834                    }
4835                    self.state.event(
4836                        "recover",
4837                        format!("judge {} voted again after the limit", j + 1),
4838                    );
4839                }
4840                Err(e) => {
4841                    self.state
4842                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4843                }
4844            }
4845        }
4846
4847        // A seat that ranked again is present even if its re-vote failed —
4848        // `tally` falls back to the initial ranking's first choice — so clear
4849        // its quota loss. Seats that still fail keep theirs and stay absent.
4850        let recovered_keys: BTreeSet<String> = recovered
4851            .iter()
4852            .map(|&j| format!("judge-{}", j + 1))
4853            .collect();
4854        self.state
4855            .quota
4856            .retain(|q| !recovered_keys.contains(&q.seat));
4857        // A seat that hit the limit again is a fresh loss, not the old one:
4858        // replace the stale entry so the history stays one-per-seat and the
4859        // daemon can tell this attempt's loss from a previous session's.
4860        for loss in judge_losses.into_iter().chain(vote_losses) {
4861            if recovered_keys.contains(&loss.seat) {
4862                continue;
4863            }
4864            self.state.quota.retain(|q| q.seat != loss.seat);
4865            self.state.quota.push(loss);
4866        }
4867
4868        // Recompute the verdict from the refreshed panel.
4869        self.state.tally = None;
4870        self.tally()?;
4871        Ok(self
4872            .state
4873            .tally
4874            .as_ref()
4875            .map(|t| t.met_quorum)
4876            .unwrap_or(false))
4877    }
4878
4879    // ---------------------------------------------------------------- graft
4880
4881    /// Port the strengths the judges attributed to the losing candidates onto
4882    /// the winner, once, between the tally and the review loop.
4883    ///
4884    /// Best-effort by construction: it returns nothing and fails nothing. A
4885    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4886    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4887    /// the run carries on as if the phase were off. See [`graft`] for the
4888    /// decisions and `AGENTS.md` for the invariants.
4889    ///
4890    /// The one exception is a rollback that could not be confirmed: the
4891    /// rejected commit may still be the winner's tip, so the run is blocked
4892    /// (resumable; the record stays `Running`, which makes a resume retry the
4893    /// restore) instead of reviewing, gating and merging it.
4894    async fn graft(&mut self) -> Result<()> {
4895        let Err(e) = self.graft_inner().await else {
4896            return Ok(());
4897        };
4898        if e.is::<crate::repo_lock::ResourceBlocked>() {
4899            return Err(e);
4900        }
4901        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4902            self.state.status = RunStatus::Blocked;
4903            self.state.event("graft", format!("blocked: {e:#}"));
4904            self.state.save()?;
4905            return Err(e);
4906        }
4907        // An error after the `Running` record was saved may have left the
4908        // fixer's work in the tree: the record must not stay unsettled
4909        // (a later resume would reset past the review fixes), so confirm the
4910        // restore now, or block.
4911        if let Some(rec) = self.state.graft.clone()
4912            && !rec.outcome.settled()
4913        {
4914            if let Some(winner) = self.state.winner().cloned() {
4915                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4916                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4917                        return Err(re);
4918                    }
4919                    self.state.status = RunStatus::Blocked;
4920                    self.state.event("graft", format!("blocked: {re:#}"));
4921                    self.state.save()?;
4922                    return Err(re);
4923                }
4924            }
4925            if let Some(g) = self.state.graft.as_mut() {
4926                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4927            }
4928        }
4929        self.state.event("graft", format!("skipped: {e:#}"));
4930        if let Err(e) = self.state.save() {
4931            tracing::warn!("save after graft: {e:#}");
4932        }
4933        Ok(())
4934    }
4935
4936    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4937    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4938    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4939        let _coordination = self.coordinate(wt).await?;
4940        crate::recovery::restore_repair(wt, pre)
4941            .await
4942            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4943    }
4944
4945    /// A skip is a run event, recorded once however often the node is re-entered.
4946    fn graft_skipped(&mut self, why: &str) {
4947        let message = format!("skipped: {why}");
4948        if !self
4949            .state
4950            .events
4951            .iter()
4952            .any(|e| e.node == "graft" && e.message == message)
4953        {
4954            self.state.event("graft", message);
4955        }
4956    }
4957
4958    async fn graft_inner(&mut self) -> Result<()> {
4959        // A round that was started and never finished (crash, kill): the
4960        // fixer may have left anything in the tree, so put the winner back.
4961        if let Some(rec) = self.state.graft.clone()
4962            && !rec.outcome.settled()
4963        {
4964            // Reviews exist only if the graft ended without its record being
4965            // settled (a swallowed error): the tree has moved on since, and
4966            // a reset to `pre` would discard the review fixes.
4967            if self.state.reviews.is_empty()
4968                && let Some(winner) = self.state.winner().cloned()
4969            {
4970                self.graft_restore(&winner.worktree, &rec.pre).await?;
4971            }
4972            if rec.outcome == GraftOutcome::WaitingForRestore {
4973                self.state.graft = None;
4974                self.state.save()?;
4975            } else {
4976                if let Some(g) = self.state.graft.as_mut() {
4977                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4978                }
4979                self.state.event(
4980                    "graft",
4981                    "rolled back: the previous round was interrupted; the winner is untouched",
4982                );
4983                self.state.save()?;
4984                return Ok(());
4985            }
4986        }
4987        let plan = match graft::plan(&self.state) {
4988            Ok(plan) => plan,
4989            Err(skip) => {
4990                if skip != graft::Skip::AlreadyDone {
4991                    self.graft_skipped(skip.reason());
4992                    self.state.save()?;
4993                }
4994                return Ok(());
4995            }
4996        };
4997        let Some(winner) = self.state.winner().cloned() else {
4998            return Ok(());
4999        };
5000        let wt = winner.worktree.clone();
5001        if !git::is_clean(&wt).await? {
5002            self.graft_skipped("the winner's worktree has uncommitted changes");
5003            self.state.save()?;
5004            return Ok(());
5005        }
5006        let pre = git::rev_parse(&wt, "HEAD").await?;
5007        // Saved before the fixer is called: a stop from here on is repaired
5008        // by the branch above on the next entry.
5009        self.state.graft = Some(GraftRecord {
5010            pre: pre.clone(),
5011            post: None,
5012            lines: None,
5013            limit: None,
5014            agent: None,
5015            outcome: GraftOutcome::Running,
5016            requested: graft::requested(&plan),
5017            adoption: None,
5018            adoption_error: None,
5019        });
5020        self.state.event(
5021            "graft",
5022            format!(
5023                "porting strengths of candidate(s) {} onto {}",
5024                plan.strengths
5025                    .iter()
5026                    .map(|(l, _, _)| l.to_string())
5027                    .collect::<Vec<_>>()
5028                    .join(", "),
5029                plan.winner
5030            ),
5031        );
5032        self.state.save()?;
5033
5034        let fix_prompt = prompt::graft(
5035            &self.state.instruction,
5036            plan.winner,
5037            &plan.strengths,
5038            &self.state.config.graph.language,
5039        );
5040        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5041        let sessions = self.state.config.graph.sessions;
5042        let artifacts = agent::artifacts_dir(&self.state.dir());
5043        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5044        let (_, seat, out) = self
5045            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5046                SeatJob {
5047                    prompt: fix_prompt.clone(),
5048                    spec,
5049                    seat,
5050                    cwd: wt.clone(),
5051                    timeout,
5052                    allow_write: true,
5053                    sessions,
5054                    artifacts: artifacts.clone(),
5055                    stem: "graft".to_owned(),
5056                    handover: None,
5057                }
5058            })
5059            .await;
5060        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5061            // Preserve the need to restore earlier fallback edits without
5062            // confusing an unstarted seat with an interrupted invocation.
5063            self.state.graft.as_mut().expect("graft record").outcome =
5064                GraftOutcome::WaitingForRestore;
5065            return self.stop_for_resource();
5066        }
5067        let mut why: Option<String> = None;
5068        match &out {
5069            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5070                why = Some("the fixer produced nothing usable".to_owned());
5071            }
5072            AgentOutcome::Ok(_) => {}
5073            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5074            AgentOutcome::Quota(o) => {
5075                self.state.quota.push(QuotaLoss::from_report(
5076                    seat.key.clone(),
5077                    "graft",
5078                    o.quota.as_ref(),
5079                ));
5080                why = Some("rate limited (quota)".to_owned());
5081            }
5082            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5083                why = Some(format!(
5084                    "the fixer failed: {}",
5085                    e.lines()
5086                        .next()
5087                        .unwrap_or("")
5088                        .chars()
5089                        .take(160)
5090                        .collect::<String>()
5091                ));
5092            }
5093        }
5094        let agent_id = seat.agent.clone();
5095        self.state.seats.insert(seat.key.clone(), seat);
5096
5097        let mut post = None;
5098        let mut lines = None;
5099        let _coordination = self.coordinate(&wt).await?;
5100        if why.is_none() {
5101            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5102            if let Ok(r) = &rescued {
5103                self.state.note_withheld("graft", &r.withheld);
5104            }
5105            let head = git::rev_parse(&wt, "HEAD").await?;
5106            // What is measured must be what gets reviewed and gated: a failed
5107            // rescue, or anything still uncommitted (withheld lockfiles
5108            // aside, which are never part of the tree), rejects the graft.
5109            let dirty = match &rescued {
5110                Ok(r) => git::status_porcelain(&wt)
5111                    .await
5112                    .map(|st| {
5113                        st.lines().any(|l| {
5114                            !r.withheld
5115                                .iter()
5116                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5117                        })
5118                    })
5119                    .unwrap_or(true),
5120                Err(_) => true,
5121            };
5122            if dirty {
5123                why = Some("the fixer left changes that could not be committed".to_owned());
5124            } else if head == pre {
5125                why = Some("the fixer changed nothing".to_owned());
5126            } else {
5127                lines = git::changed_lines(&wt, &pre, &head).await;
5128                post = Some(head);
5129            }
5130        }
5131        let (adoption, adoption_error) = match &out {
5132            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5133            _ => (None, None),
5134        };
5135        let requested = graft::requested(&plan);
5136        let outcome = match why {
5137            None => {
5138                self.state.event(
5139                    "graft",
5140                    format!(
5141                        "applied: {}",
5142                        lines.map_or_else(
5143                            || "changed lines unavailable".to_owned(),
5144                            |n| format!("{n} changed lines")
5145                        )
5146                    ),
5147                );
5148                GraftOutcome::Applied
5149            }
5150            Some(why) => {
5151                // Not settled until the restore is confirmed: the record
5152                // stays `Running`, so a resume retries it.
5153                crate::recovery::restore_repair(&wt, &pre)
5154                    .await
5155                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5156                self.state.event(
5157                    "graft",
5158                    format!("rolled back: {why}; the winner is untouched"),
5159                );
5160                GraftOutcome::RolledBack(why)
5161            }
5162        };
5163        self.state.graft = Some(GraftRecord {
5164            pre,
5165            post,
5166            lines,
5167            limit: None,
5168            agent: Some(agent_id),
5169            outcome,
5170            requested,
5171            adoption,
5172            adoption_error,
5173        });
5174        self.state.save()?;
5175        Ok(())
5176    }
5177
5178    // ----------------------------------------------------------------- fold
5179
5180    async fn fold_losers(&mut self) -> Result<()> {
5181        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5182            return Ok(());
5183        };
5184        if !self.state.candidate_preservation.is_empty() {
5185            bail!("candidate preservation incomplete; refusing cleanup");
5186        }
5187        let repo = self.state.repo.clone();
5188        let _coordination = self.coordinate(&repo).await?;
5189        let mut folded = Vec::new();
5190        for i in 0..self.state.candidates.len() {
5191            let c = &self.state.candidates[i];
5192            if c.label == winner || c.folded {
5193                continue;
5194            }
5195            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5196            remove_preserved_worktree(&repo, &wt).await?;
5197            if git::branch_exists(&repo, &branch).await?
5198                && !git::branch_delete(&repo, &branch).await?
5199            {
5200                bail!("candidate branch removal refused");
5201            }
5202            self.state.candidates[i].folded = true;
5203            self.state.save()?;
5204            folded.push(label.to_string());
5205        }
5206        // The judges are finished; their checkouts are pure cost from here.
5207        let root = self.state.worktree_root();
5208        for j in 1..=self.roles.judges.len() {
5209            let wt = root.join(format!("judge-{j}"));
5210            if wt.exists() {
5211                remove_preserved_worktree(&repo, &wt).await?;
5212            }
5213        }
5214        // The design-deliberation stage is finished by the time a tally
5215        // exists — same reasoning as the judges above.
5216        if self.state.config.graph.advise {
5217            for k in 1..=self.state.config.graph.advisors {
5218                let wt = root.join(format!("advisor-{k}"));
5219                if wt.exists() {
5220                    remove_preserved_worktree(&repo, &wt).await?;
5221                }
5222            }
5223        }
5224        if !folded.is_empty() {
5225            self.state
5226                .event("fold", format!("folded candidates {}", folded.join(", ")));
5227            self.state.save()?;
5228        }
5229        Ok(())
5230    }
5231
5232    // ------------------------------------------------------------ base sync
5233
5234    async fn hold_base_sync(&mut self) -> Result<bool> {
5235        self.hold_base_sync_as(None).await
5236    }
5237
5238    async fn hold_base_sync_kind(
5239        &mut self,
5240        kind: crate::base_sync_recovery::StopKind,
5241    ) -> Result<()> {
5242        self.hold_base_sync_as(Some(kind)).await?;
5243        Ok(())
5244    }
5245
5246    async fn hold_base_sync_as(
5247        &mut self,
5248        kind: Option<crate::base_sync_recovery::StopKind>,
5249    ) -> Result<bool> {
5250        if !self.state.base_sync_blocked() {
5251            self.state.base_sync_recovery.stop = None;
5252            return Ok(false);
5253        }
5254        let head = match self.state.winner() {
5255            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5256                .await
5257                .unwrap_or_default(),
5258            None => String::new(),
5259        };
5260        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5261            && !authorization.consumed
5262            && authorization.stop.head != head
5263        {
5264            authorization.refuse(format!(
5265                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5266                authorization.stop.head
5267            ));
5268        }
5269        self.state.record_base_sync_stop(&head);
5270        if let Some(kind) = kind {
5271            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5272            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5273                stop.kind = kind;
5274            }
5275        }
5276        self.state.save()?;
5277        Ok(true)
5278    }
5279
5280    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5281    /// anything verifies it.
5282    ///
5283    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5284    /// read whatever is checked out in the winner's worktree. Left alone that
5285    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5286    /// it when the run *branched* - and a run takes long enough that the base
5287    /// has usually moved by the time it gets here. A gate that ran there
5288    /// answers "green on the commit this run started from", not "green on
5289    /// what is about to land", and the difference showed up three times in
5290    /// one day as a green run whose merge would have reverted a file another
5291    /// pull request had already landed.
5292    ///
5293    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5294    /// `land::Step::Rebase` calls, rather than a second implementation of the
5295    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5296    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5297    /// doc warns about ("two answers to a question nobody notices until a
5298    /// diff is wrong").
5299    ///
5300    /// Foreign-checkout reviews instead merge the base into the owned shadow
5301    /// worktree, preserving the ancestry required for fast-forward delivery.
5302    /// A conflicting merge is aborted and held for operator resolution.
5303    ///
5304    /// A conflict is not the end of the road: the standing rebase is handed
5305    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5306    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5307    /// land). Once it finishes, review and the gate run as usual on the
5308    /// rebased tree, which is where a breakage the new base caused is caught
5309    /// by the ordinary gate-fix round. magi resolves nothing itself.
5310    ///
5311    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5312    /// the current durable recovery interval, bounds how often the base is
5313    /// synchronized during active recovery (lifetime attempts remain evidence).
5314    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5315    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5316    /// what was tried (rounds spent, paths still conflicted) and the branch
5317    /// and worktree stay exactly as they were - untouched, for a person to
5318    /// look at - which is also what makes re-entering this function
5319    /// afterwards a no-op instead of a second attempt at the same wall. A
5320    /// push failure ends the same way.
5321    async fn sync_to_base(&mut self) -> Result<()> {
5322        if self.state.status == RunStatus::AlreadyInBase {
5323            return Ok(());
5324        }
5325        self.hold_base_sync().await?;
5326        let conflicted = self
5327            .state
5328            .base_sync
5329            .as_ref()
5330            .is_some_and(|s| s.conflict.is_some());
5331        let Some(winner) = self.state.winner().cloned() else {
5332            return Ok(());
5333        };
5334
5335        let repo = self.state.repo.clone();
5336        let remote = self.state.config.merge.remote.clone();
5337        let base_branch = self.state.base_branch.clone();
5338        let tracking = format!("{remote}/{base_branch}");
5339
5340        let fetched_base =
5341            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5342        // No network, or the remote never had this branch: the run already
5343        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5344        // otherwise), and one that got this far is not blocked by a fetch
5345        // that fails now. It just has no newer tip to compare against.
5346        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5347            return Ok(());
5348        };
5349
5350        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5351        self.state.ensure_base_episode();
5352        let mut pending = self
5353            .state
5354            .base_sync_recovery
5355            .operations
5356            .last()
5357            .filter(|o| !o.finished)
5358            .cloned();
5359        let scratch = self.state.dir().join("base-sync");
5360        if git::rebase_in_progress(&scratch).await {
5361            // Older resource stops predate operation reservations. Import the
5362            // standing Git operation, never retarget it to today's main.
5363            if pending.is_none()
5364                && self.state.base_sync_recovery.operations.is_empty()
5365                && !conflicted
5366                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5367                    .await
5368                    .ok()
5369                    .as_deref()
5370                    == Some(head.as_str())
5371                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5372                && git::is_clean(&winner.worktree).await?
5373            {
5374                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5375                if self.state.base_sync.is_none() {
5376                    self.state.base_sync = Some(BaseSync {
5377                        tip: saved_tip.clone(),
5378                        behind: 0,
5379                        attempts: 0,
5380                        conflict: None,
5381                        already_in: None,
5382                    });
5383                }
5384                self.state.reserve_base_operation(&head, &saved_tip);
5385                self.state.event(
5386                    "base-recovery",
5387                    "imported standing legacy rebase as a spent synchronization reservation",
5388                );
5389                self.state.save()?;
5390                pending = self.state.base_sync_recovery.operations.last().cloned();
5391            }
5392            let expected = pending.as_ref().map(|op| op.tip.as_str());
5393            if expected.is_none()
5394                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5395            {
5396                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5397                if self.state.base_sync.is_none() {
5398                    self.state.base_sync = Some(BaseSync {
5399                        tip: tip.clone(),
5400                        behind: 0,
5401                        attempts: 0,
5402                        conflict: None,
5403                        already_in: None,
5404                    });
5405                }
5406                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5407                self.state.event("land", why);
5408                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5409                    .await?;
5410                return Ok(());
5411            }
5412        }
5413        if let Some(op) = &pending {
5414            // A replay may add the pinned base's commits, never foreign work.
5415            let mut replay_proven =
5416                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5417            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5418                let wait = repository_wait(&self.state, None);
5419                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5420                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5421                    Ok(true) => {
5422                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5423                        self.state.save()?;
5424                        git::worktree_remove(&repo, &scratch).await.ok();
5425                        replay_proven = true;
5426                    }
5427                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5428                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5429                }
5430            }
5431            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5432                if self.state.foreign_checkout.is_some() {
5433                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5434                        .await
5435                        .ok()
5436                        .as_deref()
5437                        == Some(op.head.as_str())
5438                        && git::rev_parse(&repo, &format!("{head}^2"))
5439                            .await
5440                            .ok()
5441                            .as_deref()
5442                            == Some(op.tip.as_str());
5443                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5444                    .await
5445                    .is_ok_and(|m| m.is_empty())
5446                    && let Ok(added) =
5447                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5448                {
5449                    replay_proven = true;
5450                    for commit in added {
5451                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5452                            replay_proven = false;
5453                            break;
5454                        }
5455                    }
5456                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5457                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5458                }
5459            }
5460            // Never retarget an operation across a crash, even if main moved.
5461            // sync_to_head resets and cleans with -x, so even ignored untracked
5462            // files must stop recovery independently of the old tracked tree.
5463            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5464                || (head != op.head
5465                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5466                        .await?
5467                        .is_empty())
5468                || (!git::is_clean(&winner.worktree).await?
5469                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5470                        .await
5471                        .is_ok_and(|o| o.ok()))
5472                || !replay_proven
5473            {
5474                let why = format!(
5475                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5476                    op.id
5477                );
5478                let sync = self
5479                    .state
5480                    .base_sync
5481                    .as_mut()
5482                    .context("reserved synchronization has no lifetime accounting")?;
5483                sync.conflict = Some(why.clone());
5484                self.state.event("land", why);
5485                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5486                    .await?;
5487                return Ok(());
5488            }
5489            if head != op.head {
5490                git::sync_to_head(&winner.worktree).await?;
5491            }
5492            tip = op.tip.clone();
5493        }
5494        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5495        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5496        if pending.is_none()
5497            && !conflicted
5498            && fetched_base
5499            && self
5500                .state
5501                .base_sync_recovery
5502                .episodes
5503                .last()
5504                .is_some_and(|e| e.boundary.is_some())
5505            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5506            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5507            && owner.runs.last() == Some(&self.state.id)
5508            && owner.status != crate::queue::TaskStatus::Done
5509            && !owner.fresh_start
5510            && owner.review_branch.is_none()
5511            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5512            && let Some(approval) = owner.resumption.as_ref()
5513            && approval.instruction == owner.instruction
5514            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5515            && git::is_clean(&winner.worktree).await?
5516            && self.state.renew_base_episode(&task, approval, &head, &tip)
5517        {
5518            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5519            self.state.save()?;
5520        }
5521
5522        // Before any rebase, and before a recorded conflict is honoured: a
5523        // branch whose change reached the base under other commit ids has
5524        // nothing to rebase and nothing to conflict with, and a run that
5525        // already stopped on that phantom conflict recovers here on resume.
5526        // `behind == 0` with head == tip is a branch the base has since taken
5527        // in whole, whether or not a conflict was ever recorded: the ancestry
5528        // proof must still run (`classify` ignores a head still on the start
5529        // commit).
5530        if (behind > 0 || conflicted || head == tip)
5531            && self
5532                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5533                .await?
5534        {
5535            return Ok(());
5536        }
5537        if conflicted && self.state.base_recovery_pending() {
5538            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5539            let owner =
5540                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5541            let owner_valid = owner.as_ref().is_ok_and(|t| {
5542                t.runs.last() == Some(&self.state.id)
5543                    && t.status != crate::queue::TaskStatus::Done
5544                    && !t.fresh_start
5545                    && t.review_branch.is_none()
5546            });
5547            let invalidation = if !owner_valid {
5548                Some("latest task ownership changed")
5549            } else if !owner.as_ref().is_ok_and(|t| {
5550                authorization
5551                    .generation
5552                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5553            }) {
5554                Some("owner approval generation changed")
5555            } else if let Some(scope) = &authorization.scope {
5556                if scope.tip != tip {
5557                    Some("fetched base changed")
5558                } else if !self
5559                    .state
5560                    .base_sync_recovery
5561                    .episodes
5562                    .last()
5563                    .is_some_and(|e| e.id == scope.episode)
5564                {
5565                    Some("active synchronization interval changed")
5566                } else if !owner
5567                    .as_ref()
5568                    .is_ok_and(|t| t.instruction == scope.instruction)
5569                {
5570                    Some("task instruction changed")
5571                } else {
5572                    None
5573                }
5574            } else {
5575                None
5576            };
5577            if let Some(reason) = invalidation {
5578                let authorization = self
5579                    .state
5580                    .base_sync_recovery
5581                    .authorizations
5582                    .last_mut()
5583                    .unwrap();
5584                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5585                self.state.save()?;
5586                return Ok(());
5587            }
5588            if !fetched_base
5589                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5590                || !git::is_clean(&winner.worktree).await?
5591                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5592            {
5593                return Ok(());
5594            }
5595        }
5596        let recovery = conflicted && self.state.consume_base_recovery(&head);
5597        if conflicted {
5598            // Consumption (including a refused changed HEAD) is durable before Git.
5599            self.state.save()?;
5600            if !recovery {
5601                return Ok(());
5602            }
5603        }
5604
5605        if behind == 0 {
5606            // A fixer-finished rebase moves the branch ref before the
5607            // winner's worktree is told (`sync_to_head` below). A run that
5608            // died in between resumes here with `behind == 0` and a tree still
5609            // holding the pre-rebase files, which review and the gate would
5610            // then read. That state is exactly: HEAD moved off the tip the
5611            // rebase started from, yet the tree is still identical to that
5612            // tip. A tree with edits of its own differs from it, so nothing
5613            // is thrown away.
5614            if let Some(from) = self
5615                .state
5616                .rebase_fixes
5617                .iter()
5618                .rev()
5619                .find_map(|r| r.from.clone())
5620                && from != head
5621                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5622                    .await
5623                    .is_ok_and(|o| o.ok())
5624            {
5625                git::sync_to_head(&winner.worktree).await?;
5626            }
5627            // An earlier attempt may have rebased the branch locally and died
5628            // before pushing it (only the fresh-rebase arm below pushes).
5629            // Publish it now, so the plain push at PR time is not refused as
5630            // a non-fast-forward. A run already holding a recorded conflict
5631            // never reaches here; that case is out of scope.
5632            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5633            if let Some(why) = &conflict {
5634                self.state.status = RunStatus::Blocked;
5635                self.state.event("land", why.clone());
5636            }
5637            self.state.base_sync = Some(BaseSync {
5638                tip,
5639                behind: 0,
5640                attempts,
5641                conflict,
5642                already_in: None,
5643            });
5644            if pending.is_some() {
5645                self.state.finish_base_operation(Some(&head));
5646            }
5647            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5648                .await?;
5649            self.state.save()?;
5650            return Ok(());
5651        }
5652
5653        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5654            let why = format!(
5655                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5656                 synchronization(s); synchronizing again would only race it",
5657                winner.branch
5658            );
5659            self.state.status = RunStatus::Blocked;
5660            self.state.base_sync = Some(BaseSync {
5661                tip,
5662                behind,
5663                attempts,
5664                conflict: Some(why.clone()),
5665                already_in: None,
5666            });
5667            self.state.event("land", why);
5668            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5669                .await?;
5670            self.state.save()?;
5671            return Ok(());
5672        }
5673
5674        if self.state.foreign_checkout.is_some() {
5675            // Delivery to the operator's checkout is fast-forward only. A
5676            // rebase here would discard its saved head from our ancestry.
5677            let wait = repository_wait(&self.state, None);
5678            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5679            if self.state.base_sync.is_none() {
5680                self.state.base_sync = Some(BaseSync {
5681                    tip: tip.clone(),
5682                    behind,
5683                    attempts: 0,
5684                    conflict: None,
5685                    already_in: None,
5686                });
5687            }
5688            self.state.reserve_base_operation(&head, &tip);
5689            self.state.save()?;
5690            let merged = git::git_raw(
5691                &winner.worktree,
5692                &[
5693                    "-c",
5694                    "core.hooksPath=/dev/null",
5695                    "merge",
5696                    "--no-ff",
5697                    "--no-autostash",
5698                    "--no-edit",
5699                    "-m",
5700                    &format!("Merge {tracking} before verifying"),
5701                    &tip,
5702                ],
5703            )
5704            .await?;
5705            let conflict = if merged.ok() {
5706                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5707                self.state.event(
5708                    "land",
5709                    format!("merged {tracking} into {} before verifying", winner.branch),
5710                );
5711                None
5712            } else {
5713                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5714                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5715                }
5716                let why = format!(
5717                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5718                    winner.branch,
5719                    merged.stderr.chars().take(600).collect::<String>()
5720                );
5721                self.state.status = RunStatus::Blocked;
5722                self.state.event("land", &why);
5723                Some(why)
5724            };
5725            let synced = conflict.is_none();
5726            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5727            self.state
5728                .finish_base_operation(synced.then_some(fresh.as_str()));
5729            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5730            self.state.base_sync = Some(BaseSync {
5731                tip,
5732                behind: if synced { 0 } else { behind },
5733                attempts,
5734                conflict,
5735                already_in: None,
5736            });
5737            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5738                .await?;
5739            self.state.save()?;
5740            return Ok(());
5741        }
5742
5743        self.state.event(
5744            "land",
5745            format!(
5746                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5747                winner.branch
5748            ),
5749        );
5750        self.state.save()?;
5751
5752        // The remote's copy of the branch, read now and only if the fetch
5753        // really succeeded (a stale tracking ref must never pin a lease). It is
5754        // pushed over after a rebase only when it is a commit this branch
5755        // already contains, by ancestry or by patch (an earlier rebase of ours
5756        // that never reached the remote): anything else is somebody else's work.
5757        let branch_tracking = format!("{remote}/{}", winner.branch);
5758        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5759        let remote_tip = if self.state.task_pr_branch().is_none()
5760            && matches!(&fetched_branch, Ok(o) if o.ok())
5761        {
5762            git::rev_parse(&repo, &branch_tracking).await.ok()
5763        } else {
5764            None
5765        };
5766        // A remote tip this branch does not contain is somebody else's work:
5767        // rebasing would leave a local tip that can never be pushed. Stop
5768        // before touching anything and say so.
5769        if let Some(theirs) = &remote_tip
5770            && !git::is_ancestor(&repo, theirs, &head).await
5771            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5772                .await
5773                .is_ok_and(|missing| missing.is_empty())
5774        {
5775            let why = format!(
5776                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5777                 them",
5778                short(theirs),
5779                winner.branch
5780            );
5781            self.state.status = RunStatus::Blocked;
5782            self.state.base_sync = Some(BaseSync {
5783                tip,
5784                behind,
5785                attempts,
5786                conflict: Some(why.clone()),
5787                already_in: None,
5788            });
5789            self.state.event("land", why);
5790            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5791                .await?;
5792            self.state.save()?;
5793            return Ok(());
5794        }
5795
5796        if self.state.base_sync.is_none() {
5797            self.state.base_sync = Some(BaseSync {
5798                tip: tip.clone(),
5799                behind,
5800                attempts: 0,
5801                conflict: None,
5802                already_in: None,
5803            });
5804        }
5805        self.state.reserve_base_operation(&head, &tip);
5806        self.state.save()?;
5807        let scratch = self.state.dir().join("base-sync");
5808        let rebased =
5809            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5810                .await
5811            {
5812                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5813                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5814                Err(e) => Err(e),
5815            };
5816        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5817        let stop_kind = match &rebased {
5818            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5819            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5820            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5821        };
5822        match rebased {
5823            Ok(None) => {
5824                // The branch ref moved, but a worktree that already had it
5825                // checked out (the winner's) was not told; sync its index and
5826                // files before anything reads them.
5827                git::sync_to_head(&winner.worktree).await?;
5828                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5829                self.state.finish_base_operation(Some(&fresh));
5830                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5831                let mut conflict = None;
5832                if let Some(pinned) = &remote_tip {
5833                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5834                    match pushed {
5835                        Ok(o) if o.ok() => self.state.event(
5836                            "land",
5837                            format!("pushed rebased {} to {remote}", winner.branch),
5838                        ),
5839                        Ok(o) => {
5840                            conflict = Some(format!(
5841                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5842                                winner.branch,
5843                                short(pinned),
5844                                o.stderr.chars().take(600).collect::<String>()
5845                            ));
5846                        }
5847                        Err(e) => {
5848                            conflict = Some(format!(
5849                                "rebased {} locally but could not push it: {e:#}",
5850                                winner.branch
5851                            ));
5852                        }
5853                    }
5854                }
5855                if let Some(why) = &conflict {
5856                    self.state.status = RunStatus::Blocked;
5857                    self.state.event("land", why.clone());
5858                }
5859                self.state.base_sync = Some(BaseSync {
5860                    tip: tip.clone(),
5861                    behind: 0,
5862                    attempts,
5863                    conflict,
5864                    already_in: None,
5865                });
5866                self.state
5867                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5868            }
5869            Ok(Some(conflict)) => {
5870                self.state.finish_base_operation(None);
5871                let why = format!(
5872                    "{} conflicts with {tracking} and did not rebase: {}",
5873                    winner.branch,
5874                    conflict.chars().take(600).collect::<String>()
5875                );
5876                self.state.status = RunStatus::Blocked;
5877                self.state.base_sync = Some(BaseSync {
5878                    tip,
5879                    behind,
5880                    attempts,
5881                    conflict: Some(why.clone()),
5882                    already_in: None,
5883                });
5884                self.state.event("land", why);
5885            }
5886            // A refused Git write or a busy repository: nothing is wrong with
5887            // the branch. The scratch rebase stays standing and the run
5888            // resumes into it (`base_sync` must not claim a conflict).
5889            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5890                self.state.save()?;
5891                return Ok(());
5892            }
5893            Err(e) => {
5894                self.state.finish_base_operation(None);
5895                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5896                self.state.status = RunStatus::Blocked;
5897                self.state.base_sync = Some(BaseSync {
5898                    tip,
5899                    behind,
5900                    attempts,
5901                    conflict: Some(why.clone()),
5902                    already_in: None,
5903                });
5904                self.state.event("land", why);
5905            }
5906        }
5907        if !self.state.base_sync_blocked() {
5908            self.state.base_sync_recovery.stop = None;
5909        }
5910        self.hold_base_sync_kind(stop_kind).await?;
5911        self.state.save()?;
5912        Ok(())
5913    }
5914
5915    /// Push a branch an earlier attempt rebased locally but never published,
5916    /// pinned to the remote tip read right after a successful fetch. Returns
5917    /// the reason when the run must stop; `None` when there was nothing to do
5918    /// (no remote copy, the same tip, or a remote copy this branch already
5919    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5920    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5921        // Task competitions publish only after the PR text gate, to their
5922        // public head; never resurrect a candidate branch on the remote.
5923        if self.state.task_pr_branch().is_some() {
5924            return None;
5925        }
5926        let repo = self.state.repo.clone();
5927        let remote = self.state.config.merge.remote.clone();
5928        let fetched = git::fetch(&repo, &remote, branch).await;
5929        if !matches!(&fetched, Ok(o) if o.ok()) {
5930            return None;
5931        }
5932        let branch_tracking = format!("{remote}/{branch}");
5933        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5934        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5935            return None;
5936        }
5937        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5938            .await
5939            .is_ok_and(|missing| missing.is_empty())
5940        {
5941            return Some(format!(
5942                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5943                 them",
5944                short(&theirs)
5945            ));
5946        }
5947        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5948            Ok(o) if o.ok() => {
5949                self.state
5950                    .event("land", format!("pushed rebased {branch} to {remote}"));
5951                None
5952            }
5953            Ok(o) => Some(format!(
5954                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5955                 someone may have pushed): {}",
5956                short(&theirs),
5957                o.stderr.chars().take(600).collect::<String>()
5958            )),
5959            Err(e) => Some(format!(
5960                "{branch} is rebased locally but could not be pushed: {e:#}"
5961            )),
5962        }
5963    }
5964
5965    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5966    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5967    /// whether it did.
5968    ///
5969    /// Checked only when the base is ahead of the branch. A failing check is
5970    /// "not proven" - the ordinary rebase path then decides - never a reason to
5971    /// stop the run.
5972    ///
5973    /// The remote copy of the branch is held to the same standard as the local
5974    /// one: if it carries a tip this worktree does not, that tip must itself be
5975    /// proven in the base, or nothing is settled (a pull request would
5976    /// otherwise be closed over commits nobody checked). The pull request is
5977    /// closed *before* the terminal status is saved; if that fails for a
5978    /// reason other than a refusal (no network, a `gh` error) the run is left
5979    /// `Blocked` with the reason as its conflict, which a resume retries -
5980    /// the same recovery a phantom conflict gets.
5981    async fn settle_already_in(
5982        &mut self,
5983        branch: &str,
5984        tip: &str,
5985        head: &str,
5986        attempts: usize,
5987        behind: usize,
5988    ) -> Result<bool> {
5989        let repo = self.state.repo.clone();
5990        let remote = self.state.config.merge.remote.clone();
5991        let start = self.state.base_commit.clone();
5992        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5993            Ok(Some(e)) => e,
5994            Ok(None) => return Ok(false),
5995            Err(e) => {
5996                tracing::warn!("already-in-base check for {branch}: {e:#}");
5997                return Ok(false);
5998            }
5999        };
6000        let mut verified = vec![head.to_owned()];
6001        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6002        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6003        if matches!(&fetched, Ok(o) if o.ok())
6004            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6005            && theirs != head
6006        {
6007            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6008                Ok(Some(_)) => verified.push(theirs),
6009                _ => return Ok(false),
6010            }
6011        }
6012        let base_branch = self.state.base_branch.clone();
6013        let message = format!(
6014            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6015             land",
6016            evidence.names(),
6017            evidence.proof.as_str()
6018        );
6019        let closed =
6020            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6021                .await;
6022        match closed {
6023            Ok(Ok(url)) => self
6024                .state
6025                .event("land", format!("closed {url}: superseded on {base_branch}")),
6026            Ok(Err(why)) => self
6027                .state
6028                .event("land", format!("did not close a pull request: {why}")),
6029            Err(e) => {
6030                let why = format!(
6031                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6032                     not be closed ({e:#}); resume to retry"
6033                );
6034                self.state.status = RunStatus::Blocked;
6035                self.state.base_sync = Some(BaseSync {
6036                    tip: tip.to_owned(),
6037                    behind,
6038                    attempts,
6039                    conflict: Some(why.clone()),
6040                    already_in: None,
6041                });
6042                self.state.event("land", why);
6043                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6044                    .await?;
6045                self.state.save()?;
6046                return Ok(true);
6047            }
6048        }
6049        self.state.status = RunStatus::AlreadyInBase;
6050        self.state.base_sync_recovery.stop = None;
6051        self.state.base_sync = Some(BaseSync {
6052            tip: tip.to_owned(),
6053            behind,
6054            attempts,
6055            conflict: None,
6056            already_in: Some(evidence),
6057        });
6058        self.state.event("land", message);
6059        self.state.save()?;
6060        self.settle_questions();
6061        Ok(true)
6062    }
6063
6064    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6065    /// last landed the winner on, once it has run, else the commit the run
6066    /// branched from.
6067    ///
6068    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6069    /// and `vote` all happen before there is a winner to rebase, so they
6070    /// compare every candidate against the branch point on purpose, and a
6071    /// base that moves after they are already done cannot change an answer
6072    /// they already gave.
6073    fn landing_base(&self) -> String {
6074        self.state
6075            .base_sync
6076            .as_ref()
6077            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6078    }
6079
6080    // ------------------------------------------------------- operator fix
6081
6082    /// Route specific, already-recorded review findings to a fixer for a
6083    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6084    /// entry point.
6085    ///
6086    /// Distinct from `review_loop`'s own fix step in three ways: it never
6087    /// runs a reviewer wave, it never spends review-round budget, and what
6088    /// happened is recorded as an [`OperatorFixRequest`] appended to
6089    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6090    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6091    /// and vote must never be rewritten to look like a manufactured blocking
6092    /// verdict.
6093    ///
6094    /// Only meaningful once review has actually concluded: `Ready` (handed
6095    /// off with findings still open, or simply concluded clean while minor
6096    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6097    /// gate failed). Everything else is refused: a run still in progress
6098    /// should simply be resumed, and a `Merged` run's branch has already
6099    /// landed — reopening *this* run's own record cannot change that, so the
6100    /// answer there is a fresh `magi review <branch>`.
6101    ///
6102    /// A real commit here re-verifies through a fresh, ordinary review-only
6103    /// run on the same branch ([`Self::review`]) rather than reopening this
6104    /// run's own `review_loop`: once any round in this run's history went
6105    /// clean, `review_conclusion` treats that as permanent by design (the
6106    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6107    /// way to force one more genuine reviewer wave out of *this* run without
6108    /// either rewriting history or weakening that guarantee for every other
6109    /// caller. A review-only run costs nothing extra — no implementation, no
6110    /// judging, no vote — and exercises the exact same review → verify →
6111    /// gate → (human) merge path, unmodified.
6112    pub async fn fix_selected(
6113        &mut self,
6114        ids: &[String],
6115        reason: &str,
6116        allow_stale: bool,
6117    ) -> Result<()> {
6118        let reason = reason.trim();
6119        if reason.is_empty() {
6120            bail!("a fix request needs a reason — that is the operator's own record of why");
6121        }
6122        if ids.is_empty() {
6123            bail!("no finding id given");
6124        }
6125        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6126            bail!(
6127                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6128                 has already concluded — can be given a targeted fix. A run still \
6129                 in progress should simply be resumed; a `merged` run's branch has \
6130                 already landed, so its answer is a fresh `magi review <branch>`, \
6131                 not reopening this run's own record",
6132                self.state.id,
6133                self.state.status.as_str()
6134            );
6135        }
6136        let Some(winner) = self.state.winner().cloned() else {
6137            bail!("run {} has no winning candidate to fix", self.state.id);
6138        };
6139        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6140            bail!(
6141                "branch `{}` no longer exists; this run cannot be extended",
6142                winner.branch
6143            );
6144        }
6145        let home = crate::run::home();
6146        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6147            bail!(
6148                "run {} is currently being worked on by another magi process",
6149                self.state.id
6150            );
6151        }
6152        // Held for the rest of this call, including the follow-up review
6153        // below: two `magi fix` invocations against the same run must not
6154        // both reach the worktree manipulation further down, which would
6155        // otherwise race to remove and recreate the same directory — see
6156        // [`FixClaim`]'s own doc.
6157        let _claim = FixClaim::acquire(&self.state.dir())?;
6158
6159        // Resolve every id before spending anything — an unknown id refuses
6160        // the whole request rather than silently dropping it — and dedup
6161        // while keeping the operator's own order.
6162        let mut seen = BTreeSet::new();
6163        let mut findings = Vec::new();
6164        let mut missing = Vec::new();
6165        for id in ids {
6166            if !seen.insert(id.clone()) {
6167                continue;
6168            }
6169            match self.state.finding(id) {
6170                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6171                    id: f.id.clone(),
6172                    severity: f.severity,
6173                    reviewer_vote: rec.vote,
6174                    round: round.round,
6175                    round_head: round.head.clone(),
6176                    reviewer: rec.reviewer,
6177                    agent: rec.agent.clone(),
6178                    file: f.file.clone(),
6179                    line: f.line,
6180                    title: f.title.clone(),
6181                    detail: f.detail.clone(),
6182                    outcome: OperatorFixOutcome::Pending,
6183                }),
6184                None => missing.push(id.clone()),
6185            }
6186        }
6187        if !missing.is_empty() {
6188            bail!(
6189                "unknown finding id(s): {}; nothing was changed",
6190                missing.join(", ")
6191            );
6192        }
6193
6194        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6195        let stale_details: Vec<(String, String)> = findings
6196            .iter()
6197            .filter(|f| f.round_head != head_at_request)
6198            .map(|f| (f.id.clone(), f.round_head.clone()))
6199            .collect();
6200        let stale = !stale_details.is_empty();
6201        if stale && !allow_stale {
6202            bail!(
6203                "the branch has moved since some finding(s) were raised — {} — now \
6204                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6205                stale_details
6206                    .iter()
6207                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6208                    .collect::<Vec<_>>()
6209                    .join(", "),
6210                short(&head_at_request)
6211            );
6212        }
6213
6214        let request = OperatorFixRequest {
6215            requested_at: Timestamp::now(),
6216            reason: reason.to_owned(),
6217            findings,
6218            head_at_request: head_at_request.clone(),
6219            allow_stale,
6220            stale,
6221            fix: None,
6222            result_head: None,
6223            follow_up_review_run: None,
6224        };
6225        self.state.event(
6226            "fix",
6227            format!(
6228                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6229                request.findings.len(),
6230                request
6231                    .findings
6232                    .iter()
6233                    .map(|f| f.id.as_str())
6234                    .collect::<Vec<_>>()
6235                    .join(", "),
6236            ),
6237        );
6238        // Recorded now, before any worktree work or the fixer call itself —
6239        // and re-saved at each checkpoint below: a crash at any point after
6240        // this (mid fixer call, mid follow-up review) must not lose the fact
6241        // that this was requested, for which findings, and why. Everything
6242        // past this point reads and writes through `request_index` rather
6243        // than a local variable, since `request` itself is moved here.
6244        self.state.operator_fixes.push(request);
6245        self.state.save()?;
6246        let request_index = self.state.operator_fixes.len() - 1;
6247
6248        // A fresh, dedicated worktree for this one call, never the winner's
6249        // own worktree in place: that one may already be gone (folded away),
6250        // and reusing it in place would leave the branch checked out there
6251        // when the follow-up review below tries to check it out again. Freed
6252        // immediately after, either way — but only once confirmed clean:
6253        // `worktree_remove` is a `git worktree remove --force`, which would
6254        // otherwise discard uncommitted work left there by the operator or
6255        // another process before this had a chance to even look at it.
6256        let repo = self.state.repo.clone();
6257        let coordination = self.coordinate(&repo).await?;
6258        if winner.worktree.exists() {
6259            // Lockfiles a rescue commit withheld stay untracked on purpose and
6260            // are already recorded; they are not the operator's work to protect.
6261            let dirty = git::git(
6262                &winner.worktree,
6263                &["status", "--porcelain", "--untracked-files=all"],
6264            )
6265            .await?;
6266            let only_withheld = dirty.lines().all(|l| {
6267                l.strip_prefix("?? ")
6268                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6269            });
6270            if !only_withheld {
6271                bail!(
6272                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6273                     discard them first",
6274                    winner.worktree.display()
6275                );
6276            }
6277            git::worktree_remove(&self.state.repo, &winner.worktree)
6278                .await
6279                .ok();
6280        }
6281        let fix_worktree = self.state.worktree_root().join("operator-fix");
6282        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6283        git::git(
6284            &self.state.repo,
6285            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6286        )
6287        .await
6288        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6289        if !git::is_clean(&fix_worktree).await? {
6290            git::worktree_remove(&self.state.repo, &fix_worktree)
6291                .await
6292                .ok();
6293            bail!(
6294                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6295                winner.branch
6296            );
6297        }
6298        let repo_for_setup = self.state.repo.clone();
6299        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6300            Ok(()) => {
6301                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6302            }
6303            Err(e) => Err(e),
6304        };
6305        if let Err(e) = setup {
6306            git::worktree_remove(&self.state.repo, &fix_worktree)
6307                .await
6308                .ok();
6309            return Err(e);
6310        }
6311
6312        // This explicit operator request retries availability independently
6313        // of execute(); the previous call has already saved and cleaned up.
6314        self.state.resource_block = None;
6315
6316        let run_id = self.state.id.clone();
6317        let prompts = self.state.config.prompts.clone();
6318        let language = self.state.config.graph.language.clone();
6319        let sessions = self.state.config.graph.sessions;
6320        let artifacts = agent::artifacts_dir(&self.state.dir());
6321        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6322            .findings
6323            .iter()
6324            .map(|f| Finding {
6325                id: f.id.clone(),
6326                severity: f.severity,
6327                file: f.file.clone(),
6328                line: f.line,
6329                title: f.title.clone(),
6330                detail: f.detail.clone(),
6331            })
6332            .collect();
6333        let fix_prompt = prompt::operator_fix_in(
6334            &self.state.instruction,
6335            &finding_list,
6336            reason,
6337            &stale_details,
6338            &head_at_request,
6339            &language,
6340            &self.state.config.graph.github_language,
6341        );
6342        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6343        drop(coordination);
6344        let (job, seat, out) = self
6345            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6346                prompt: fix_prompt.clone(),
6347                spec,
6348                seat,
6349                cwd: fix_worktree.clone(),
6350                timeout,
6351                allow_write: true,
6352                sessions,
6353                artifacts: artifacts.clone(),
6354                stem: "operator-fix".to_owned(),
6355                handover: None,
6356            })
6357            .await;
6358        let agent_id = seat.agent.clone();
6359
6360        let mut fix = FixRecord {
6361            agent: agent_id,
6362            addressed: Vec::new(),
6363            rejected: Vec::new(),
6364            notes: String::new(),
6365            committed: false,
6366            failed: None,
6367            duration_ms: 0,
6368            continuation: None,
6369        };
6370        let mut final_seat = seat.clone();
6371        match out {
6372            AgentOutcome::Ok(o) => {
6373                fix.duration_ms = o.duration_ms;
6374                let parsed = verdict::extract_json::<FixReport>(&o.text);
6375                let incomplete_reason = match &parsed {
6376                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6377                        "the reply parsed, but it reported a command whose own CLI \
6378                         never confirmed an exit status"
6379                            .to_owned(),
6380                    ),
6381                    Ok(_) => None,
6382                    Err(e) => Some(e.to_string()),
6383                };
6384                match incomplete_reason {
6385                    None => {
6386                        let report = parsed.expect("checked Ok above");
6387                        fix.addressed = report.addressed;
6388                        fix.rejected = report.rejected;
6389                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6390                    }
6391                    Some(reason) => {
6392                        let (resumed_seat, resolved, failure, cont) = self
6393                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6394                            .await;
6395                        fix.duration_ms += cont.cumulative_wait_ms;
6396                        fix.continuation = Some(cont);
6397                        final_seat = resumed_seat;
6398                        match resolved {
6399                            Some(report) => {
6400                                fix.addressed = report.addressed;
6401                                fix.rejected = report.rejected;
6402                                fix.notes =
6403                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6404                            }
6405                            None => fix.failed = failure,
6406                        }
6407                    }
6408                }
6409            }
6410            AgentOutcome::Dropped(o) => {
6411                fix.duration_ms = o.duration_ms;
6412                let why = o
6413                    .dropped
6414                    .as_ref()
6415                    .map(|d| d.why.as_str())
6416                    .unwrap_or("the CLI ended the stream without delivering its answer");
6417                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6418            }
6419            AgentOutcome::Quota(o) => {
6420                self.state.quota.push(QuotaLoss::from_report(
6421                    final_seat.key.clone(),
6422                    "fix",
6423                    o.quota.as_ref(),
6424                ));
6425                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6426            }
6427            AgentOutcome::Failed(e) => fix.failed = Some(e),
6428            AgentOutcome::ResourceBlocked(e) => {
6429                fix.failed = Some(format!("the fixer was not started: {e}"));
6430            }
6431        }
6432        // Even a deferred call must rescue edits, release the worktree and
6433        // finalize its record below so an explicit `magi fix` can retry.
6434        if fix.continuation.is_none() {
6435            fix.continuation = Some(ContinuationRecord::not_needed());
6436        }
6437        self.state.seats.insert(final_seat.key.clone(), final_seat);
6438
6439        let rescue_message = format!(
6440            "magi: operator-selected fix ({}) (uncommitted work)",
6441            self.state.operator_fixes[request_index]
6442                .findings
6443                .iter()
6444                .map(|f| f.id.as_str())
6445                .collect::<Vec<_>>()
6446                .join(", ")
6447        );
6448        let repo = self.state.repo.clone();
6449        let coordination = self.coordinate(&repo).await?;
6450        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6451            self.state.note_withheld("fix", &r.withheld);
6452        }
6453        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6454        fix.committed = after != head_at_request;
6455        git::worktree_remove(&self.state.repo, &fix_worktree)
6456            .await
6457            .ok();
6458        drop(coordination);
6459
6460        self.state.event(
6461            "fix",
6462            match &fix.failed {
6463                Some(reason) => format!(
6464                    "operator fix: adoption report was lost ({reason}); {}",
6465                    if fix.committed {
6466                        "committed"
6467                    } else {
6468                        "NO new commit"
6469                    }
6470                ),
6471                None => format!(
6472                    "operator fix: {} addressed, {} rejected, {}",
6473                    fix.addressed.len(),
6474                    fix.rejected.len(),
6475                    if fix.committed {
6476                        "committed"
6477                    } else {
6478                        "NO new commit"
6479                    }
6480                ),
6481            },
6482        );
6483
6484        // Every selected finding gets an outcome — never left `Pending` once
6485        // the fixer's own turn is over. A report that never came back at all
6486        // marks every one of them `Unreported`, not silently "not addressed":
6487        // quota, a dropped stream, or an exhausted continuation are gaps in
6488        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6489        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6490        for f in &mut self.state.operator_fixes[request_index].findings {
6491            f.outcome = if fix.failed.is_some() {
6492                OperatorFixOutcome::Unreported
6493            } else if fix.addressed.contains(&f.id) {
6494                OperatorFixOutcome::Addressed
6495            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6496                OperatorFixOutcome::Rejected { why: r.why.clone() }
6497            } else {
6498                OperatorFixOutcome::Unreported
6499            };
6500        }
6501
6502        let committed = fix.committed;
6503        if committed {
6504            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6505        }
6506        self.state.operator_fixes[request_index].fix = Some(fix);
6507        // Saved again now that the fixer's own outcome is final, on top of
6508        // the save right after the request was first pushed above.
6509        self.state.save()?;
6510        if self.state.resource_block.is_some() {
6511            return self.stop_for_resource();
6512        }
6513
6514        if committed {
6515            self.state.event(
6516                "fix",
6517                format!(
6518                    "operator fix committed {}; opening a follow-up review-only run",
6519                    short(&after)
6520                ),
6521            );
6522            // The operator asked for the fix, and the follow-up serves whatever
6523            // task the run it follows served.
6524            let origin =
6525                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6526            match Self::review(
6527                &self.state.repo,
6528                &winner.branch,
6529                self.state.config.clone(),
6530                origin,
6531            )
6532            .await
6533            {
6534                Ok(mut follow_up) => {
6535                    follow_up.state.event(
6536                        "start",
6537                        format!(
6538                            "requested by an operator fix on run {} for finding(s) {}",
6539                            self.state.id,
6540                            self.state.operator_fixes[request_index]
6541                                .findings
6542                                .iter()
6543                                .map(|f| f.id.as_str())
6544                                .collect::<Vec<_>>()
6545                                .join(", "),
6546                        ),
6547                    );
6548                    follow_up.state.save()?;
6549                    let follow_up_id = follow_up.state.id.clone();
6550                    // The follow-up is a run like any other: it belongs to the
6551                    // task of the run it follows, or to one filed for it.
6552                    let adopted = match crate::direct::adopt(
6553                        &follow_up.state,
6554                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6555                    ) {
6556                        Ok(a) => a,
6557                        Err(e) => {
6558                            // An ownerless run must not spend agent calls; it
6559                            // stays saved, and `magi run --resume` adopts it.
6560                            self.state.event(
6561                                "fix",
6562                                format!(
6563                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6564                                ),
6565                            );
6566                            self.state.save()?;
6567                            return Ok(());
6568                        }
6569                    };
6570                    let executed = follow_up.execute().await;
6571                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6572                    if let Some(a) = adopted {
6573                        a.finish(&follow_up.state, executed);
6574                    }
6575                    if let Some(e) = failure {
6576                        self.state.event(
6577                            "fix",
6578                            format!(
6579                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6580                            ),
6581                        );
6582                    }
6583                    self.state.operator_fixes[request_index].follow_up_review_run =
6584                        Some(follow_up_id);
6585                }
6586                Err(e) => {
6587                    self.state.event(
6588                        "fix",
6589                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6590                    );
6591                }
6592            }
6593            self.state.save()?;
6594        }
6595
6596        Ok(())
6597    }
6598
6599    // --------------------------------------------------------------- review
6600
6601    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6602    /// names, in order, each at most once, moving on only when the call
6603    /// advances (an error, a quota hit or nothing usable - the same decision
6604    /// point `agent::chain_advances` is for the other chained roles).
6605    ///
6606    /// Each agent gets its own seat from `Runner::seat`: the same agent
6607    /// continues its conversation (the winner's own implementer seat when it
6608    /// is the winner's author, now that the competition is over), another
6609    /// takes a fresh one so the full prompt is sent again. A handover is
6610    /// recorded under `node`, which is what makes the fallback stick for the
6611    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6612    /// is returned, so an exhausted chain reads exactly like a single failed
6613    /// fixer: one quota loss, the same wording, the same refund. An earlier
6614    /// attempt's edits are left in the tree and judged, with the final
6615    /// attempt's, by what git says afterwards.
6616    ///
6617    /// Returns the job that produced the outcome, for `continue_fix_report`.
6618    async fn ask_fixer(
6619        &mut self,
6620        winner: &Candidate,
6621        node: &'static str,
6622        round: Option<usize>,
6623        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6624    ) -> (SeatJob, SeatState, AgentOutcome) {
6625        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6626        self.ask_fixer_among(attempts, node, round, None, build)
6627            .await
6628    }
6629
6630    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6631    /// worktree and a commit: before the next agent is asked after one that
6632    /// did not satisfy the call, the worktree is put back on that commit, so
6633    /// a failed agent's partial edits are not carried into the next one.
6634    async fn ask_fixer_among(
6635        &mut self,
6636        attempts: Vec<(AgentSpec, String)>,
6637        node: &'static str,
6638        round: Option<usize>,
6639        restore: Option<(&Path, &str)>,
6640        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6641    ) -> (SeatJob, SeatState, AgentOutcome) {
6642        let run_id = self.state.id.clone();
6643        let prompts = self.state.config.prompts.clone();
6644        let cache = self.state.config.cache_dir();
6645        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6646        let mut last = None;
6647        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6648            let seat = self.seat(&key, &spec.id);
6649            let mut job = build(spec.clone(), seat);
6650            if i > 0 {
6651                job.stem = format!("{}-{}", job.stem, spec.id);
6652            }
6653            let ctx = WaveCtx {
6654                carry_seats: false,
6655                pauses: Some((&self.pause, &self.interrupt)),
6656                run: &run_id,
6657                node,
6658                prompts: &prompts,
6659                cache: cache.as_deref(),
6660                round,
6661            };
6662            let (seat, out) =
6663                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6664            // A resource block never advances the chain: the chosen agent
6665            // was not asked, so it must be the one asked again on resume.
6666            let advances = match &out {
6667                AgentOutcome::Ok(o) => agent::output_advances(o),
6668                AgentOutcome::ResourceBlocked(_) => false,
6669                _ => true,
6670            };
6671            if let Some(next) = ids.get(i + 1)
6672                && advances
6673            {
6674                self.state.seats.insert(seat.key.clone(), seat.clone());
6675                let class =
6676                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6677                let reason = match &out {
6678                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6679                    other => fail_reason(other),
6680                };
6681                // A failed agent's edits must not reach the next one, and a
6682                // restore that did not work ends the chain: the call fails
6683                // and the caller's own (confirmed) restore decides.
6684                if let Some((wt, rev)) = restore
6685                    && let Err(e) = self.graft_restore(wt, rev).await
6686                {
6687                    let why =
6688                        format!("could not restore the worktree before the next fixer: {e:#}");
6689                    tracing::warn!("{why}");
6690                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6691                        AgentOutcome::ResourceBlocked(why)
6692                    } else {
6693                        AgentOutcome::Failed(why)
6694                    };
6695                    last = Some((job, seat, out));
6696                    break;
6697                }
6698                record_handover(
6699                    &mut self.state,
6700                    node,
6701                    &seat.key,
6702                    &spec.id,
6703                    next,
6704                    &class,
6705                    &reason,
6706                );
6707                continue;
6708            }
6709            last = Some((job, seat, out));
6710            break;
6711        }
6712        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6713        if let AgentOutcome::Quota(output) = &out
6714            && node != "graft"
6715        {
6716            // Graft is best effort: its caller records the loss and restores
6717            // the pre-graft tree, then proceeds to review and gate.
6718            // Every fallback was tried. Machine outages cannot be repaired by
6719            // spending a fix round; use each caller's resumable resource path.
6720            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6721            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6722            self.state.quota.push(loss);
6723            self.state.seats.insert(seat.key.clone(), seat.clone());
6724            self.state.status = RunStatus::Blocked;
6725            return (
6726                job,
6727                seat,
6728                AgentOutcome::ResourceBlocked(
6729                    "fixer unavailable; restore agent availability and resume".into(),
6730                ),
6731            );
6732        }
6733        (job, seat, out)
6734    }
6735
6736    /// The one repair a verification-only failure earns (see
6737    /// [`crate::recovery`]), before the first review round of a run that took
6738    /// a branch over.
6739    ///
6740    /// Saved as started *before* the fixer is called, so a resume never
6741    /// repairs twice; a start found on re-entry is rolled back to the commit
6742    /// it began from. A failed or unusable fixer leaves the tree as it was and
6743    /// the review goes on over it, so the first round's own verification
6744    /// reports the failure again. The failure is never carried into a round as
6745    /// a result: verification is re-run on whatever commit this leaves.
6746    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6747        use crate::recovery::{Mode, Repair};
6748        if !self.state.reviews.is_empty() {
6749            return Ok(());
6750        }
6751        let Some(mut rec) = self.state.recovery.clone() else {
6752            return Ok(());
6753        };
6754        if rec.mode != Mode::VerifyRepair {
6755            return Ok(());
6756        }
6757        let wt = winner.worktree.clone();
6758        let mut head = git::rev_parse(&wt, "HEAD").await?;
6759        match rec.repair.clone() {
6760            Repair::Done { .. } => return Ok(()),
6761            Repair::WaitingForRestore { before } => {
6762                self.graft_restore(&wt, &before).await?;
6763                head = before;
6764                rec.repair = Repair::NotStarted;
6765                self.state.recovery = Some(rec.clone());
6766                self.state.save()?;
6767            }
6768            Repair::Started { before } => {
6769                // Interrupted: the unfinished edits are not trusted.
6770                // An unconfirmed rollback blocks the run with `Started` kept,
6771                // so a resume retries it; the tree is never reviewed as is.
6772                if let Err(e) = self.graft_restore(&wt, &before).await {
6773                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6774                        return Err(e);
6775                    }
6776                    return self
6777                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6778                        .await;
6779                }
6780                rec.repair = Repair::Done {
6781                    before: before.clone(),
6782                    after: before,
6783                    committed: false,
6784                    failed: Some("interrupted; rolled back".to_owned()),
6785                };
6786                self.state
6787                    .event("recovery", "the repair was interrupted and rolled back");
6788                self.state.recovery = Some(rec);
6789                self.state.save()?;
6790                return Ok(());
6791            }
6792            Repair::NotStarted => {}
6793        }
6794        if crate::recovery::recheck(&mut rec, &head) {
6795            self.state.event("recovery", rec.summary());
6796            self.state.recovery = Some(rec);
6797            self.state.save()?;
6798            return Ok(());
6799        }
6800        rec.repair = Repair::Started {
6801            before: head.clone(),
6802        };
6803        self.state.recovery = Some(rec.clone());
6804        self.state.save()?;
6805
6806        let language = self.state.config.graph.language.clone();
6807        let sessions = self.state.config.graph.sessions;
6808        let artifacts = agent::artifacts_dir(&self.state.dir());
6809        let fix_prompt = prompt::recovery_fix(
6810            &self.state.instruction,
6811            &rec,
6812            self.state.config.graph.review_rounds,
6813            &language,
6814        );
6815        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6816        self.state.event(
6817            "recovery",
6818            format!(
6819                "repair: the fixer starts from the failed verification on {}",
6820                short(&head)
6821            ),
6822        );
6823        let (_job, seat, out) = self
6824            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6825                prompt: fix_prompt.clone(),
6826                spec,
6827                seat,
6828                cwd: wt.clone(),
6829                timeout,
6830                allow_write: true,
6831                sessions,
6832                artifacts: artifacts.clone(),
6833                stem: "fix-recovery".to_owned(),
6834                handover: None,
6835            })
6836            .await;
6837        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6838            rec.repair = Repair::WaitingForRestore { before: head };
6839            self.state.recovery = Some(rec);
6840            return self.stop_for_resource();
6841        }
6842        let failed = match &out {
6843            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6844            AgentOutcome::Quota(o) => {
6845                self.state.quota.push(QuotaLoss::from_report(
6846                    seat.key.clone(),
6847                    "fix",
6848                    o.quota.as_ref(),
6849                ));
6850                Some("rate limited (quota); the fixer could not run".to_owned())
6851            }
6852            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6853            AgentOutcome::Failed(e) => Some(e.clone()),
6854        };
6855        self.state.seats.insert(seat.key.clone(), seat);
6856        let coordination = self.coordinate(&wt).await?;
6857        let failed = match failed {
6858            // The agent's partial edits are not kept.
6859            Some(why) => {
6860                if let Err(e) = git::restore_to(&wt, &head).await {
6861                    return self
6862                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6863                        .await;
6864                }
6865                Some(why)
6866            }
6867            None => {
6868                if let Ok(r) = git::rescue_commit(
6869                    &wt,
6870                    "magi: recovery repair of the failed verification (uncommitted work)",
6871                )
6872                .await
6873                {
6874                    self.state.note_withheld("fix", &r.withheld);
6875                }
6876                None
6877            }
6878        };
6879        let after = git::rev_parse(&wt, "HEAD").await?;
6880        drop(coordination);
6881        self.state.event(
6882            "recovery",
6883            match &failed {
6884                Some(why) => format!("repair: nothing kept ({why})"),
6885                None if after != head => {
6886                    format!("repair: committed {}", short(&after))
6887                }
6888                None => "repair: the fixer made no commit".to_owned(),
6889            },
6890        );
6891        rec.repair = Repair::Done {
6892            before: head.clone(),
6893            committed: after != head,
6894            after,
6895            failed,
6896        };
6897        self.state.recovery = Some(rec);
6898        self.state.save()?;
6899        Ok(())
6900    }
6901
6902    /// A repair whose rollback could not be confirmed: the run is blocked and
6903    /// resumable, and the unconfirmed state is never reviewed.
6904    async fn stop_recovery(
6905        &mut self,
6906        mut rec: crate::recovery::RecoveryRecord,
6907        head: &str,
6908        why: String,
6909    ) -> Result<()> {
6910        // `Started` stays recorded: a resume retries the restore.
6911        rec.repair = crate::recovery::Repair::Started {
6912            before: head.to_owned(),
6913        };
6914        self.state.recovery = Some(rec);
6915        self.state
6916            .event("recovery", format!("repair could not be undone: {why}"));
6917        self.state.status = RunStatus::Blocked;
6918        self.state.save()?;
6919        bail!("recovery repair could not be rolled back: {why}")
6920    }
6921
6922    async fn review_loop(&mut self) -> Result<()> {
6923        if self.hold_base_sync().await? {
6924            return Ok(());
6925        }
6926        loop {
6927            if self
6928                .state
6929                .base_sync
6930                .as_ref()
6931                .is_some_and(|s| s.conflict.is_some())
6932            {
6933                return Ok(());
6934            }
6935            if self.catchup_repairable() {
6936                let record = self.state.reviews.last().expect("repairable round").clone();
6937                let winner = self
6938                    .state
6939                    .winner()
6940                    .cloned()
6941                    .context("catch-up repair has no winner")?;
6942                if !self.fix_review_round(&winner, record).await? {
6943                    return Ok(());
6944                }
6945            }
6946            self.review_loop_pass().await?;
6947            if !self.catchup_repairable() {
6948                return Ok(());
6949            }
6950        }
6951    }
6952
6953    /// Derive an owed catch-up repair from durable round evidence, including
6954    /// reentry after a crash before the fixer was dispatched.
6955    fn catchup_repairable(&self) -> bool {
6956        self.state.reviews.len() < self.state.config.graph.review_rounds
6957            && self.state.reviews.last().is_some_and(|r| {
6958                !r.clean
6959                    && r.blocking == 0
6960                    && r.fix.is_none()
6961                    && r.e2e_status() == E2eStatus::Failed
6962            })
6963    }
6964
6965    async fn review_loop_pass(&mut self) -> Result<()> {
6966        // A base that would not rebase is a person's decision, not a review
6967        // round: nothing here would change the answer, and reviewers and a
6968        // fixer would be spending real budget on a tree that cannot land
6969        // regardless of what they find.
6970        if self
6971            .state
6972            .base_sync
6973            .as_ref()
6974            .is_some_and(|s| s.conflict.is_some())
6975        {
6976            return Ok(());
6977        }
6978        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6979        // agent files with `magi task add` name the run that paid for it. The
6980        // prompt overlay is cloned alongside it because the waves borrow it
6981        // while `self` is mutably borrowed by the node's own bookkeeping.
6982        let run_id = self.state.id.clone();
6983        let prompts = self.state.config.prompts.clone();
6984        let Some(winner) = self.state.winner().cloned() else {
6985            return Ok(());
6986        };
6987        let granted_review = self
6988            .state
6989            .repair_grants
6990            .last()
6991            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6992            .cloned();
6993        let fresh_review = granted_review
6994            .as_ref()
6995            .is_some_and(|g| self.state.reviews.len() == g.review_start);
6996        let max_rounds = granted_review
6997            .as_ref()
6998            .map_or(self.state.config.graph.review_rounds, |g| {
6999                g.review_start.saturating_add(1)
7000            });
7001        // A clean round, an exhausted round budget, or a stalled tree (see
7002        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7003        // they are recorded — recomputed here, not read off `status`, so a
7004        // reentry into a run that already stopped restates the identical
7005        // verdict instead of silently handing back whatever an earlier node
7006        // in this same walk clobbered `status` to (a solo-candidate
7007        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7008        // below runs an empty range once the budget is spent, and would
7009        // otherwise fall through without touching `status` at all.
7010        if let Some(status) =
7011            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7012        {
7013            // A reentry after a crash between the last round's save and
7014            // `stop_reviewing` reaches the hand-off here, not there.
7015            if status == RunStatus::Gating {
7016                self.record_contested_handoff();
7017            }
7018            self.state.status = status;
7019            self.state.save()?;
7020            return Ok(());
7021        }
7022        self.state.status = RunStatus::Reviewing;
7023        // A last recorded round whose own verification never resolved
7024        // (`ResourceBlocked` — the shared build cache, not the patch) is
7025        // never a concluded round, whatever the round budget says: starting
7026        // a fresh round on top of it would spend a whole new reviewer wave
7027        // re-reading an unchanged patch instead of just retrying the one
7028        // check that actually needs it, and once the budget is spent the
7029        // loop below has nothing left to do at all (its range is empty).
7030        // Retry that check directly instead, exactly the same retry
7031        // `stop_reviewing` already does for its own catch-up case.
7032        if !fresh_review
7033            && self.state.reviews.last().is_some_and(|r| {
7034                r.e2e_status() == E2eStatus::ResourceBlocked
7035                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7036            })
7037        {
7038            let shell = self.state.config.shell();
7039            return self
7040                .stop_unresolved_review(
7041                    "the last round's own verification never resolved",
7042                    &shell,
7043                    &winner.worktree,
7044                )
7045                .await;
7046        }
7047
7048        if granted_review.is_none() {
7049            self.recovery_repair(&winner).await?;
7050        }
7051        if self.resource_halt()? {
7052            return Ok(());
7053        }
7054
7055        let repo = self.state.repo.clone();
7056        let root = self.state.worktree_root();
7057        let language = self.state.config.graph.language.clone();
7058        let sessions = self.state.config.graph.sessions;
7059        let artifacts = self.merge_repair_artifacts();
7060        let base = self.landing_base();
7061        let base_short = short(&base);
7062        let reviewers = self.roles.reviewers.clone();
7063        let shell = self.state.config.shell();
7064
7065        for round in (self.state.reviews.len() + 1)..=max_rounds {
7066            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7067            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7068            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7069            // The prior round's own record, already persisted — never a
7070            // hand-carried variable of just its failing output: that is
7071            // exactly what let a round's e2e result drift out of sync with
7072            // which commit it was actually about (see `SCHEMA`'s doc for
7073            // schema 8). Judged against `head`, the commit reviewers are
7074            // about to look at now, so the summary always reads as "an
7075            // earlier head" here — this round's own patch has not been
7076            // checked yet.
7077            let prev_verification = self
7078                .state
7079                .reviews
7080                .last()
7081                .and_then(|r| r.verification_summary(&head));
7082            // Only the first round of a run that took a branch over: later
7083            // rounds have this run's own history.
7084            let prior_section = match (&self.state.recovery, round) {
7085                (Some(rec), 1) if self.state.reviews.is_empty() => {
7086                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7087                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7088                            Some(crate::recovery::head_of(
7089                                &git::diff(&winner.worktree, reviewed, "HEAD")
7090                                    .await
7091                                    .unwrap_or_default(),
7092                                crate::recovery::SINCE_DIFF_MAX,
7093                            ))
7094                        }
7095                        _ => None,
7096                    };
7097                    Some(prompt::recovery_review(rec, since.as_deref()))
7098                }
7099                _ => None,
7100            };
7101
7102            // Each reviewer gets its own detached checkout of exactly this
7103            // commit: nobody can perturb the winner's tree, and the fixer can
7104            // keep working without racing a reviewer.
7105            let mut jobs = Vec::new();
7106            let coordination = self.coordinate(&repo).await?;
7107            for (r, spec) in reviewers.iter().cloned().enumerate() {
7108                let wt = root.join(format!("review-{}", r + 1));
7109                // `clean -fdx` in a reset wipes setup's products, so every
7110                // round sets the seat up again.
7111                if wt.exists() {
7112                    git::reset_detached(&wt, &head).await?;
7113                } else {
7114                    git::worktree_add_detached(&repo, &wt, &head).await?;
7115                }
7116                self.setup_seat_worktree(&repo, &wt, false).await?;
7117                let seat_key = format!("review-{}", r + 1);
7118                // The seat starts the round on whoever answered it last, not
7119                // on the agent the spec names, so a failure is not re-paid.
7120                let spec = pick_start_spec(
7121                    &self.roles.reviewer_roster,
7122                    spec,
7123                    self.state.seat_history.get(&seat_key),
7124                );
7125                let seat = self.seat(&seat_key, &spec.id);
7126                jobs.push(SeatJob {
7127                    prompt: prompt::review(&prompt::ReviewCtx {
7128                        instruction: &self.state.instruction,
7129                        branch: &winner.branch,
7130                        base_short: &base_short,
7131                        stat: &stat,
7132                        patch: &patch,
7133                        verification: prev_verification.as_ref(),
7134                        prior: prior_section.as_deref(),
7135                        reviewers: reviewers.len(),
7136                        round,
7137                        rounds: max_rounds,
7138                        // A review-only run has no rankings, so nothing
7139                        // competed for this patch and the reviewer is told so.
7140                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7141                        lens: Lens::for_seat(r),
7142                        language: &language,
7143                        github_language: &self.state.config.graph.github_language,
7144                    }),
7145                    spec,
7146                    seat,
7147                    cwd: wt,
7148                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7149                    allow_write: false,
7150                    sessions,
7151                    artifacts: artifacts.clone(),
7152                    stem: format!("review-{round}-{}", r + 1),
7153                    handover: None,
7154                });
7155            }
7156
7157            drop(coordination);
7158            self.state.event(
7159                "review",
7160                format!(
7161                    "round {round}: {} reviewers on {}",
7162                    jobs.len(),
7163                    short(&head)
7164                ),
7165            );
7166            let mut quota_losses = Vec::new();
7167            let review_retries = self.state.config.graph.retries;
7168            let review_cache = self.state.config.cache_dir();
7169            let ctx = WaveCtx {
7170                carry_seats: true,
7171                pauses: Some((&self.pause, &self.interrupt)),
7172                run: &run_id,
7173                node: "review",
7174                prompts: &prompts,
7175                cache: review_cache.as_deref(),
7176                round: Some(round),
7177            };
7178            let results = ask_json_wave::<Review>(
7179                jobs,
7180                Arc::clone(&self.sem),
7181                review_retries,
7182                &self.roles.reviewer_roster,
7183                &ctx,
7184                &mut quota_losses,
7185                &mut self.state,
7186                &|_: &Review| Ok(()),
7187            )
7188            .await;
7189            if self.state.resource_block.is_some() {
7190                return self.stop_for_resource();
7191            }
7192            // Counted before the move below: how many of *this* round's
7193            // reviewer seats were lost to their own rate limit, as opposed to
7194            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7195            let round_quota_missing = quota_losses.len();
7196            self.state.quota.extend(quota_losses);
7197
7198            let mut records = Vec::new();
7199            let mut all_findings = Vec::new();
7200            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7201                let agent_id = seat.agent.clone();
7202                self.state.seats.insert(seat.key.clone(), seat);
7203                let mut record = ReviewRecord {
7204                    reviewer: r + 1,
7205                    agent: agent_id,
7206                    summary: String::new(),
7207                    findings: Vec::new(),
7208                    vote: None,
7209                    failed: None,
7210                    duration_ms: 0,
7211                    // Set for both outcomes: `failed: Some(_)` with
7212                    // `attempts > 0` is a seat every retry still lost, not a
7213                    // recovered one — only `failed: None` with `attempts > 0`
7214                    // reads as "answered after a nudge" (see this field's own
7215                    // doc).
7216                    attempts,
7217                };
7218                match res {
7219                    Ok((review, out)) => {
7220                        // Sanitized here, at the point every other piece of
7221                        // agent prose in this file is (candidate summaries,
7222                        // deliberation turns, vote reasons): a reviewer's own
7223                        // words are the one thing about it that could name
7224                        // it, and reconsideration below broadcasts this same
7225                        // summary and these same findings to every other
7226                        // seat on the panel.
7227                        record.summary =
7228                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7229                        record.vote = Some(review.vote);
7230                        record.duration_ms = out.duration_ms;
7231                        for (n, mut f) in review.findings.into_iter().enumerate() {
7232                            // ids are magi's, never the agent's: the fixer's
7233                            // adoption report is keyed by them.
7234                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7235                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7236                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7237                            // `file` is agent-supplied prose too, never
7238                            // checked against the real tree — the same
7239                            // exposure `title`/`detail` above have, just in
7240                            // a field easy to forget because it looks like a
7241                            // path rather than free text.
7242                            f.file = f
7243                                .file
7244                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7245                            all_findings.push(f.clone());
7246                            record.findings.push(f);
7247                        }
7248                        self.state.event(
7249                            "review",
7250                            format!(
7251                                "round {round}: reviewer {} voted {} with {} finding(s)",
7252                                r + 1,
7253                                review.vote.label(),
7254                                record.findings.len()
7255                            ),
7256                        );
7257                    }
7258                    Err(e) => {
7259                        record.failed = Some(e.to_string());
7260                        self.state.event(
7261                            "review",
7262                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7263                        );
7264                    }
7265                }
7266                records.push(record);
7267            }
7268
7269            // Tally the round's votes and, if they split, spend the one
7270            // round of reconsideration the split -> deliberate -> revote
7271            // shape `judge`/`vote` use for the panel, sized down to what a
7272            // read-only review round can afford: one round, and a revote
7273            // rather than an argument, because the panel already wrote its
7274            // reasoning down as findings the first time around.
7275            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7276            let vote_split =
7277                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7278            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7279            if vote_split {
7280                self.state.event(
7281                    "review",
7282                    format!(
7283                        "round {round}: votes split ({}) — one round of reconsideration",
7284                        initial_votes
7285                            .iter()
7286                            .map(|v| v.label())
7287                            .collect::<Vec<_>>()
7288                            .join(", ")
7289                    ),
7290                );
7291                // Seats read every seat's findings and votes, still numbered
7292                // and never named — the same anonymity `review` itself keeps.
7293                let panel: Vec<ReviewSeatReport<'_>> = records
7294                    .iter()
7295                    .filter_map(|r| {
7296                        r.vote.map(|vote| ReviewSeatReport {
7297                            reviewer: r.reviewer,
7298                            vote,
7299                            summary: &r.summary,
7300                            findings: &r.findings,
7301                        })
7302                    })
7303                    .collect();
7304
7305                let mut jobs = Vec::new();
7306                let mut seats_at = Vec::new();
7307                for (r, spec) in reviewers.iter().cloned().enumerate() {
7308                    // A seat with no initial vote has nothing to reconsider
7309                    // from and stays absent, the same as it stayed absent
7310                    // from `panel` above.
7311                    if records[r].vote.is_none() {
7312                        continue;
7313                    }
7314                    let wt = root.join(format!("review-{}", r + 1));
7315                    let seat_key = format!("review-{}", r + 1);
7316                    let spec = self.occupant(&seat_key, spec);
7317                    let seat = self.seat(&seat_key, &spec.id);
7318                    // A seat with no live session has already forgotten the
7319                    // initial review's prompt — restate the patch it is
7320                    // voting on, the same as `deliberate`/`vote` do for a
7321                    // judge in the same position.
7322                    // The panel already carries this seat's own review and
7323                    // vote, so restating the patch makes the prompt whole for
7324                    // a seat handed to another agent.
7325                    let build = |with_patch: bool| {
7326                        prompt::review_reconsider(&ReviewReconsiderCtx {
7327                            instruction: &self.state.instruction,
7328                            reviewer: r + 1,
7329                            lens: Lens::for_seat(r),
7330                            panel: &panel,
7331                            patch: with_patch.then_some(ReviewPatch {
7332                                branch: &winner.branch,
7333                                base_short: &base_short,
7334                                stat: &stat,
7335                                patch: &patch,
7336                            }),
7337                            round,
7338                            rounds: max_rounds,
7339                            language: &language,
7340                        })
7341                    };
7342                    let full = build(true);
7343                    let prompt = if has_context(&spec, &seat, sessions) {
7344                        build(false)
7345                    } else {
7346                        full.clone()
7347                    };
7348                    jobs.push(SeatJob {
7349                        prompt,
7350                        spec,
7351                        seat,
7352                        cwd: wt,
7353                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7354                        allow_write: false,
7355                        sessions,
7356                        artifacts: artifacts.clone(),
7357                        stem: format!("review-{round}-reconsider-{}", r + 1),
7358                        handover: Some(full),
7359                    });
7360                    seats_at.push(r);
7361                }
7362
7363                let mut recon_quota_losses = Vec::new();
7364                let recon_cache = self.state.config.cache_dir();
7365                let recon_ctx = WaveCtx {
7366                    carry_seats: true,
7367                    pauses: Some((&self.pause, &self.interrupt)),
7368                    run: &run_id,
7369                    node: "review",
7370                    prompts: &prompts,
7371                    cache: recon_cache.as_deref(),
7372                    round: Some(round),
7373                };
7374                let recon_results = ask_json_wave::<ReviewRevote>(
7375                    jobs,
7376                    Arc::clone(&self.sem),
7377                    review_retries,
7378                    &self.roles.reviewer_roster,
7379                    &recon_ctx,
7380                    &mut recon_quota_losses,
7381                    &mut self.state,
7382                    &|_: &ReviewRevote| Ok(()),
7383                )
7384                .await;
7385                if self.state.resource_block.is_some() {
7386                    return self.stop_for_resource();
7387                }
7388                self.state.quota.extend(recon_quota_losses);
7389
7390                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7391                    let agent_id = seat.agent.clone();
7392                    self.state.seats.insert(seat.key.clone(), seat);
7393                    let mut rec = ReviewRevoteRecord {
7394                        reviewer: r + 1,
7395                        agent: agent_id,
7396                        vote: None,
7397                        reason: String::new(),
7398                        failed: None,
7399                    };
7400                    match res {
7401                        Ok((rv, _)) => {
7402                            rec.vote = Some(rv.vote);
7403                            rec.reason =
7404                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7405                            self.state.event(
7406                                "review",
7407                                format!(
7408                                    "round {round}: reviewer {} revoted {}",
7409                                    r + 1,
7410                                    rv.vote.label()
7411                                ),
7412                            );
7413                        }
7414                        Err(e) => {
7415                            rec.failed = Some(e.to_string());
7416                            self.state.event(
7417                                "review",
7418                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7419                            );
7420                        }
7421                    }
7422                    reconsideration.push(rec);
7423                }
7424            } else if initial_votes.len() > 1 {
7425                self.state.event(
7426                    "review",
7427                    format!(
7428                        "round {round}: votes agreed ({}) — no reconsideration",
7429                        initial_votes[0].label()
7430                    ),
7431                );
7432            }
7433
7434            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7435            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7436            // A round that already has a blocking finding and a round left to
7437            // try is going back to the fixer no matter what `verify.e2e`
7438            // says, so running it first only spends the loop's slowest step
7439            // (minutes, for a Rust repo's full test suite) on a head about
7440            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7441            // runs once a round has no blocking findings left (see
7442            // `round_is_clean`, which a deferred — empty — `e2e` can never
7443            // satisfy since `blocking` is nonzero whenever this branch is
7444            // taken), and `stop_reviewing` forces a real run before it will
7445            // ever read a deferred round as green.
7446            let defer_e2e =
7447                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7448            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7449                let reason =
7450                    format!("{blocking} blocking finding(s) already required a fix this round");
7451                self.state.event(
7452                    "verify",
7453                    format!(
7454                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7455                         {}); it will run once a round has none left",
7456                        short(&head)
7457                    ),
7458                );
7459                (Vec::new(), false, true, Some(reason))
7460            } else {
7461                let e2e_commands = self.state.config.verify.e2e.clone();
7462                let cache_dir = self.state.config.cache_dir();
7463                let context = format!("round {round}");
7464                let wait = self.repository_wait();
7465                let (e2e, verify_retried) = with_cache_lease(
7466                    &mut self.state,
7467                    cache_dir.as_deref(),
7468                    "e2e",
7469                    "e2e",
7470                    &winner.worktree,
7471                    &head,
7472                    verify_timeout,
7473                    &context,
7474                    &wait,
7475                    |state, budget| {
7476                        let shell = shell.clone();
7477                        let e2e_commands = e2e_commands.clone();
7478                        let worktree = winner.worktree.clone();
7479                        let context = context.clone();
7480                        async move {
7481                            run_e2e_with_retry(
7482                                state,
7483                                &shell,
7484                                &e2e_commands,
7485                                &worktree,
7486                                budget,
7487                                &context,
7488                            )
7489                            .await
7490                        }
7491                    },
7492                )
7493                .await?;
7494                (e2e, verify_retried, false, None)
7495            };
7496
7497            let expected = records.len();
7498            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7499            let incomplete = answered < expected;
7500            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7501            let policy = self.state.config.graph.incomplete_review;
7502            let clean = round_is_clean(
7503                blocking,
7504                e2e_ok,
7505                answered,
7506                expected,
7507                round_quota_missing,
7508                policy,
7509            );
7510
7511            let mut round_record = ReviewRound {
7512                round,
7513                head: head.clone(),
7514                verified_head: None,
7515                verified_at: None,
7516                reviews: records,
7517                e2e,
7518                verify_retried,
7519                e2e_deferred,
7520                e2e_defer_reason,
7521                fix: None,
7522                blocking,
7523                answered,
7524                expected,
7525                clean,
7526                progressed: false,
7527                vote_split,
7528                reconsideration,
7529                verdict: None,
7530            };
7531            // The final vote per seat is its revote where reconsideration
7532            // ran and answered, its initial vote otherwise — the same
7533            // fallback `tally` uses for a judge whose private vote failed.
7534            round_record.verdict = ReviewVote::worst(
7535                round_record
7536                    .final_votes()
7537                    .into_iter()
7538                    .map(|(_, _, vote)| vote),
7539            );
7540            // Which commit and when magi actually attempted to check —
7541            // known the moment a command was dispatched against `head`,
7542            // whether or not it finished: a resource-blocked attempt still
7543            // targeted a specific commit at a specific time, and leaving
7544            // that unrecorded is exactly what made `verification_summary`
7545            // report a fresh attempt as "commit unknown ... recorded before
7546            // this was tracked", indistinguishable from a genuinely old,
7547            // untracked record. Only a deferred or unconfigured round never
7548            // ran at all and has nothing to record — see
7549            // `ReviewRound::verified_head`'s own doc.
7550            if !matches!(
7551                round_record.e2e_status(),
7552                E2eStatus::Deferred | E2eStatus::NotConfigured
7553            ) {
7554                round_record.verified_head = Some(head.clone());
7555                round_record.verified_at = Some(Timestamp::now());
7556            }
7557
7558            if incomplete {
7559                let missing: Vec<String> = round_record
7560                    .reviews
7561                    .iter()
7562                    .filter(|r| r.failed.is_some())
7563                    .map(|r| format!("review-{}", r.reviewer))
7564                    .collect();
7565                self.state.event(
7566                    "review",
7567                    format!(
7568                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7569                        missing.join(", ")
7570                    ),
7571                );
7572            }
7573
7574            if clean {
7575                self.state.event(
7576                    "review",
7577                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7578                        format!(
7579                            "round {round}: clean (warn policy, incomplete panel) — no \
7580                             blocking findings from the seats that answered, verification green"
7581                        )
7582                    } else if incomplete {
7583                        format!(
7584                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7585                             quorum) — no blocking findings from the seats that answered, \
7586                             verification green",
7587                            expected - answered
7588                        )
7589                    } else {
7590                        format!("round {round}: clean — no blocking findings, verification green")
7591                    },
7592                );
7593                self.state.reviews.push(round_record);
7594                self.state.status = RunStatus::Gating;
7595                self.state.save()?;
7596                return Ok(());
7597            }
7598
7599            // Nothing was raised and verification passed, but not every seat
7600            // answered and `round_is_clean` still refused to call it clean —
7601            // either a seat is missing for a reason other than its own quota
7602            // (a crash, a timeout, unparsable output — worth another try), or
7603            // every seat that could have answered lost its quota and nobody
7604            // is left to decide on: re-review rather than send the fixer
7605            // after a round with nothing to fix.
7606            if incomplete && blocking == 0 && e2e_ok {
7607                self.state.reviews.push(round_record);
7608                self.state.save()?;
7609                if round == max_rounds {
7610                    self.state.status = RunStatus::Blocked;
7611                    self.state.event(
7612                        "review",
7613                        format!(
7614                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7615                             refusing to call it clean",
7616                            expected - answered
7617                        ),
7618                    );
7619                    return Ok(());
7620                }
7621                continue;
7622            }
7623
7624            // Nothing for the fixer to act on (`blocking == 0`) and the only
7625            // reason this round is not clean is that magi itself never got
7626            // a command to run — the shared build cache, not the patch (see
7627            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7628            // the fixer would invite a change to appease contention that has
7629            // nothing to do with the diff, and would leave this attempt
7630            // sitting in the next round's prompt as if it were about an
7631            // earlier, superseded commit rather than what it actually is:
7632            // the same head, still waiting to be checked. Wait for it the
7633            // same way the final round's own contention is already handled,
7634            // whatever round this happens to be.
7635            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7636                self.state.reviews.push(round_record);
7637                return self
7638                    .stop_unresolved_review(
7639                        "the round's own verification could not run",
7640                        &shell,
7641                        &winner.worktree,
7642                    )
7643                    .await;
7644            }
7645
7646            if round == max_rounds {
7647                self.state.reviews.push(round_record);
7648                return self
7649                    .stop_reviewing(
7650                        &format!(
7651                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7652                        ),
7653                        &shell,
7654                        &winner.worktree,
7655                    )
7656                    .await;
7657            }
7658
7659            if !self.fix_review_round(&winner, round_record).await? {
7660                return Ok(());
7661            }
7662        }
7663        Ok(())
7664    }
7665
7666    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7667    /// False means admission or continuation policy stopped the review loop.
7668    async fn fix_review_round(
7669        &mut self,
7670        winner: &Candidate,
7671        mut round_record: ReviewRound,
7672    ) -> Result<bool> {
7673        let round = round_record.round;
7674        let shell = self.state.config.shell();
7675        let step = self.run_fix_round(winner, &mut round_record).await?;
7676        if matches!(step, FixStep::Resource) {
7677            self.stop_for_resource()?;
7678            return Ok(false);
7679        }
7680        if self.state.reviews.len() < round {
7681            self.state.reviews.push(round_record);
7682        } else {
7683            self.state.reviews[round - 1] = round_record;
7684        }
7685        self.state.save()?;
7686
7687        // The fixer's own report never came back this round, even after
7688        // `continue_fix_report`'s own budget was spent on it — not an
7689        // ordinary "no report" (dropped stream, quota, plain failure),
7690        // which already reads that way and is left to the existing round
7691        // budget. Stopping here, rather than opening another round, is
7692        // what keeps a next reviewer/fixer wave from ever being
7693        // dispatched onto `winner.worktree` while whatever the seat's
7694        // last call may still have running there is unaccounted for: no
7695        // process liveness check exists (and none is being added — see
7696        // AGENTS.md/this task's own scope), so the only way to honour
7697        // "nothing starts before a valid report returns" is to not start
7698        // anything further on this worktree from this run at all.
7699        if matches!(step, FixStep::Recorded { report_lost: true }) {
7700            self.stop_reviewing(
7701                "the fixer's adoption report never came back, even after resuming its \
7702                         own seat; refusing to start another round against the same worktree \
7703                         while that is unresolved",
7704                &shell,
7705                &winner.worktree,
7706            )
7707            .await?;
7708            return Ok(false);
7709        }
7710
7711        let streak = self
7712            .state
7713            .reviews
7714            .iter()
7715            .rev()
7716            .take_while(|r| !r.progressed)
7717            .count();
7718        if streak >= STAGNANT_LIMIT {
7719            self.stop_reviewing(
7720                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7721                &shell,
7722                &winner.worktree,
7723            )
7724            .await?;
7725            return Ok(false);
7726        }
7727        Ok(true)
7728    }
7729
7730    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7731    async fn run_fix_round(
7732        &mut self,
7733        winner: &Candidate,
7734        round_record: &mut ReviewRound,
7735    ) -> Result<FixStep> {
7736        let round = round_record.round;
7737        let max_rounds = self.state.config.graph.review_rounds;
7738        let run_id = self.state.id.clone();
7739        let prompts = self.state.config.prompts.clone();
7740        let language = self.state.config.graph.language.clone();
7741        let sessions = self.state.config.graph.sessions;
7742        let artifacts = self.merge_repair_artifacts();
7743        let base = self.landing_base();
7744        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7745        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7746        let this_round_verification = round_record.verification_summary(&head);
7747        let all_findings: Vec<_> = round_record
7748            .reviews
7749            .iter()
7750            .flat_map(|r| r.findings.iter().cloned())
7751            .collect();
7752        // Fix. The winner's own implementer seat continues its conversation:
7753        // the competition is over, so context is pure benefit now.
7754        let blocking_findings: Vec<_> = all_findings
7755            .iter()
7756            .filter(|f| f.severity.blocks())
7757            .cloned()
7758            .collect();
7759        let fix_prompt = prompt::fix_in(
7760            &self.state.instruction,
7761            &blocking_findings,
7762            this_round_verification.as_ref(),
7763            round,
7764            max_rounds,
7765            &language,
7766            &self.state.config.graph.github_language,
7767        );
7768        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7769        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7770        let (job, seat, out) = self
7771            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7772                prompt: fix_prompt.clone(),
7773                spec,
7774                seat,
7775                cwd: winner.worktree.clone(),
7776                timeout,
7777                allow_write: true,
7778                sessions,
7779                artifacts: artifacts.clone(),
7780                stem: format!("fix-{round}"),
7781                handover: None,
7782            })
7783            .await;
7784        // Not asked: the round is not recorded (so a resume repeats it),
7785        // no fixer was handed over to, and the seat is left as it was.
7786        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7787            return Ok(FixStep::Resource);
7788        }
7789        let agent_id = seat.agent.clone();
7790
7791        let mut fix = FixRecord {
7792            agent: agent_id,
7793            addressed: Vec::new(),
7794            rejected: Vec::new(),
7795            notes: String::new(),
7796            committed: false,
7797            failed: None,
7798            duration_ms: 0,
7799            continuation: None,
7800        };
7801        let mut continuation = ContinuationRecord::not_needed();
7802        let mut final_seat = seat.clone();
7803        match out {
7804            AgentOutcome::Ok(o) => {
7805                fix.duration_ms = o.duration_ms;
7806                let parsed = verdict::extract_json::<FixReport>(&o.text);
7807                // A parsed report standing next to a command this same
7808                // reply's own CLI never confirmed the exit status of is
7809                // not a resolved answer — the identical `CommandEvidence`
7810                // `state.jobs` renders, read here instead of only on
7811                // display, per the completion judgment and the shown
7812                // record needing to agree.
7813                let incomplete_reason = match &parsed {
7814                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7815                        "the reply parsed, but it reported a command whose own CLI never \
7816                             confirmed an exit status"
7817                            .to_owned(),
7818                    ),
7819                    Ok(_) => None,
7820                    Err(e) => Some(e.to_string()),
7821                };
7822                match incomplete_reason {
7823                    None => {
7824                        let report = parsed.expect("checked Ok above");
7825                        fix.addressed = report.addressed;
7826                        fix.rejected = report.rejected;
7827                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7828                    }
7829                    Some(reason) => {
7830                        let (resumed_seat, resolved, failure, cont) = self
7831                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7832                            .await;
7833                        fix.duration_ms += cont.cumulative_wait_ms;
7834                        continuation = cont;
7835                        final_seat = resumed_seat;
7836                        match resolved {
7837                            Some(report) => {
7838                                fix.addressed = report.addressed;
7839                                fix.rejected = report.rejected;
7840                                fix.notes =
7841                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7842                            }
7843                            None => fix.failed = failure,
7844                        }
7845                    }
7846                }
7847            }
7848            // The CLI's raw error JSON is not a fix report to parse.
7849            AgentOutcome::Dropped(o) => {
7850                fix.duration_ms = o.duration_ms;
7851                let why = o
7852                    .dropped
7853                    .as_ref()
7854                    .map(|d| d.why.as_str())
7855                    .unwrap_or("the CLI ended the stream without delivering its answer");
7856                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7857            }
7858            AgentOutcome::Quota(o) => {
7859                self.state.quota.push(QuotaLoss::from_report(
7860                    final_seat.key.clone(),
7861                    "fix",
7862                    o.quota.as_ref(),
7863                ));
7864                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7865            }
7866            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7867        }
7868        if self.state.resource_block.is_some() {
7869            self.state.seats.insert(final_seat.key.clone(), final_seat);
7870            return Ok(FixStep::Resource);
7871        }
7872        fix.continuation = Some(continuation);
7873        self.state.seats.insert(final_seat.key.clone(), final_seat);
7874        let coordination = self.coordinate(&winner.worktree).await?;
7875        if let Ok(r) = git::rescue_commit(
7876            &winner.worktree,
7877            &format!("magi: review round {round} fixes (uncommitted work)"),
7878        )
7879        .await
7880        {
7881            self.state.note_withheld("fix", &r.withheld);
7882        }
7883        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7884        fix.committed = after != before;
7885        // Judged by what `git` says moved against base, never by the
7886        // fixer's own `addressed`/`rejected` count — see
7887        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7888        // `patch` snapshot above: swallowing this error would default
7889        // `diff_after` to empty, which almost always differs from a
7890        // non-empty `patch` and reads as "progressed" — exactly backwards
7891        // for a `git` failure the stagnation check cannot see through.
7892        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7893        drop(coordination);
7894        let progressed = diff_after != patch;
7895        let commit_note = if fix.committed {
7896            "committed"
7897        } else {
7898            "NO new commit"
7899        };
7900        let tree_note = if progressed {
7901            "changed vs base"
7902        } else {
7903            "unchanged vs base"
7904        };
7905        self.state.event(
7906            "fix",
7907            match &fix.failed {
7908                // Distinct on purpose from "0 addressed, 0 rejected": the
7909                // fixer's own diff still landed (blocking counts do keep
7910                // falling round over round), only its adoption report did
7911                // not come back, so this must never read like every
7912                // finding was reviewed and declined.
7913                Some(reason) => {
7914                    format!(
7915                        "round {round}: fixer's adoption report was lost ({reason}); \
7916                             {commit_note}, tree {tree_note}"
7917                    )
7918                }
7919                None => format!(
7920                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7921                         {tree_note}{}",
7922                    fix.addressed.len(),
7923                    fix.rejected.len(),
7924                    if continuation.outcome == ContinuationOutcome::Resumed {
7925                        format!(
7926                            " (adoption report recovered after {} continuation(s))",
7927                            continuation.attempts
7928                        )
7929                    } else {
7930                        String::new()
7931                    },
7932                ),
7933            },
7934        );
7935        round_record.fix = Some(fix);
7936        round_record.progressed = progressed;
7937        Ok(FixStep::Recorded {
7938            report_lost: matches!(
7939                continuation.outcome,
7940                ContinuationOutcome::Exhausted
7941                    | ContinuationOutcome::QuotaLost
7942                    | ContinuationOutcome::NoSession
7943            ),
7944        })
7945    }
7946
7947    /// Decide, from the last recorded round's own verification, whether
7948    /// stopping the review loop is a hand-off or a genuine block.
7949    ///
7950    /// Called once the loop has given up trying — the round budget is spent,
7951    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7952    /// findings still open, never while a round is still clean or the
7953    /// incomplete-panel case handled inline above. Gate and e2e are facts
7954    /// about the tree; a lingering review finding is an opinion, and this
7955    /// workload's own `magi stats` puts reviewer precision low enough
7956    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7957    /// must not by itself stand between a green, verified change and the
7958    /// human who decides what to do with it. A red e2e is not an opinion, so
7959    /// that case still blocks, with the failing command and a tail of its
7960    /// output recorded here rather than left in `run.json` for someone to go
7961    /// find.
7962    ///
7963    /// A round that deferred its own e2e (see [`Config::graph`]'s
7964    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7965    /// only because nothing ran, and treating an empty list as a passing one
7966    /// here is exactly the "deferred painted green" bug this function exists
7967    /// to not have. When the last round's own verification never resolved —
7968    /// deferred on purpose, or a real attempt the shared build cache blocked
7969    /// — this makes (or retries) the real run, on the actual worktree this
7970    /// loop is about to stop touching, before deciding anything. A
7971    /// resource-blocked attempt is likewise never read as either green or
7972    /// red: it is evidence about the machine, not the patch (see
7973    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7974    /// blocked cache leaves this call without deciding rather than guessing
7975    /// — the caller retries on a later reentry.
7976    /// Record, once, that the review loop handed off over a blocking finding
7977    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7978    /// `land` asks the owner even with `land_approval` off. Called from every
7979    /// path that concludes `Gating`; a reentry keeps the first record.
7980    fn record_contested_handoff(&mut self) {
7981        if self.state.contested_handoff.is_some() {
7982            return;
7983        }
7984        let Some(contested) = self
7985            .state
7986            .reviews
7987            .last()
7988            .and_then(ReviewRound::contested_handoff)
7989        else {
7990            return;
7991        };
7992        self.state.event(
7993            "review",
7994            format!(
7995                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7996                 wait for the owner's approval",
7997                contested.findings.len(),
7998                contested.rejecters.len()
7999            ),
8000        );
8001        self.state.contested_handoff = Some(contested);
8002    }
8003
8004    async fn stop_unresolved_review(
8005        &mut self,
8006        why: &str,
8007        shell: &[String],
8008        worktree: &Path,
8009    ) -> Result<()> {
8010        self.stop_reviewing_inner(why, shell, worktree, true).await
8011    }
8012
8013    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8014        self.stop_reviewing_inner(why, shell, worktree, false).await
8015    }
8016
8017    async fn stop_reviewing_inner(
8018        &mut self,
8019        why: &str,
8020        shell: &[String],
8021        worktree: &Path,
8022        repairable: bool,
8023    ) -> Result<()> {
8024        let round_idx = self.state.reviews.len() - 1;
8025        // A deferred round and a resource-blocked one are the same shape
8026        // here: neither has a real result yet, and both get one more
8027        // attempt. Read off `e2e_status` — the single source for this —
8028        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8029        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8030        // still retries instead of being read as a settled result the
8031        // instant it stops being empty.
8032        let needs_catchup_run = matches!(
8033            self.state.reviews[round_idx].e2e_status(),
8034            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8035        );
8036        if needs_catchup_run {
8037            let round = self.state.reviews[round_idx].round;
8038            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8039            let commands = self.state.config.verify.e2e.clone();
8040            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8041            let cache_dir = self.state.config.cache_dir();
8042            let context = format!(
8043                "round {round}: verification unresolved, catching up before the final decision"
8044            );
8045            let wait = self.repository_wait();
8046            let (outcomes, verify_retried) = with_cache_lease(
8047                &mut self.state,
8048                cache_dir.as_deref(),
8049                "e2e",
8050                "e2e",
8051                worktree,
8052                &attempted_head,
8053                timeout,
8054                &context,
8055                &wait,
8056                |state, budget| {
8057                    let shell = shell.to_vec();
8058                    let commands = commands.clone();
8059                    let context = context.clone();
8060                    async move {
8061                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8062                            .await
8063                    }
8064                },
8065            )
8066            .await?;
8067            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8068                self.state.record_base_progress(&attempted_head);
8069            }
8070            let last = &mut self.state.reviews[round_idx];
8071            last.e2e = outcomes;
8072            last.verify_retried = verify_retried;
8073            // Always the commit and time this attempt actually targeted,
8074            // whether or not it happens to equal the reviewed `head` and
8075            // whether or not a command finished — see
8076            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8077            // attempt is recorded too, so a later reader sees "attempted
8078            // again at T2" rather than silence.
8079            last.verified_head = Some(attempted_head);
8080            last.verified_at = Some(Timestamp::now());
8081            if verify_inconclusive(&last.e2e) {
8082                // Still not a real result: `e2e_deferred` is left exactly
8083                // as it was, so `needs_catchup_run` above reads
8084                // `ResourceBlocked` (via `e2e_status`, which checks
8085                // `resource_blocked` before `e2e_deferred`) and retries
8086                // again on the next reentry, rather than recording
8087                // contention as a red e2e and blocking the run on it.
8088                self.state.save()?;
8089                return Ok(());
8090            }
8091            last.e2e_deferred = false;
8092            // A complete panel may have been saved as unclean solely because
8093            // verification could not run. Refresh that verdict with the new
8094            // outcomes before a granted repair consumes the round's result.
8095            // Incomplete panels retain their original policy/quota decision.
8096            if last.answered == last.expected {
8097                last.clean = round_is_clean(
8098                    last.blocking,
8099                    last.e2e.iter().all(CommandOutcome::ok),
8100                    last.answered,
8101                    last.expected,
8102                    0,
8103                    self.state.config.graph.incomplete_review,
8104                );
8105            }
8106        }
8107        let last = &self.state.reviews[round_idx];
8108        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8109
8110        match last.e2e_status() {
8111            E2eStatus::Failed => {
8112                let red: Vec<String> = last
8113                    .e2e
8114                    .iter()
8115                    .filter(|o| !o.ok())
8116                    .map(|o| {
8117                        format!(
8118                            "`{}` -> {:?}\n{}",
8119                            o.command,
8120                            o.code,
8121                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8122                        )
8123                    })
8124                    .collect();
8125                let can_repair = last.blocking == 0 && last.fix.is_none();
8126                self.state.reviews[round_idx].clean = false;
8127                self.state
8128                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8129                self.state.status = RunStatus::Blocked;
8130                if repairable
8131                    && can_repair
8132                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8133                {
8134                    self.state.status = RunStatus::Reviewing;
8135                }
8136            }
8137            // `needs_catchup_run` above already retried once this call; if
8138            // it is still blocked, this is magi's own admission it could
8139            // not get a command to run, never a verdict on the patch — the
8140            // run is left exactly where a later reentry can retry again.
8141            E2eStatus::ResourceBlocked => {
8142                self.state.event(
8143                    "review",
8144                    format!(
8145                        "{why}; e2e could not run (resource unavailable); not \
8146                         deciding yet"
8147                    ),
8148                );
8149            }
8150            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8151                self.state.event(
8152                    "review",
8153                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8154                );
8155                self.record_contested_handoff();
8156                self.state.status = RunStatus::Gating;
8157            }
8158        }
8159        self.state.save()?;
8160        Ok(())
8161    }
8162
8163    // ----------------------------------------------------------------- gate
8164
8165    async fn gate(&mut self) -> Result<()> {
8166        if self.hold_base_sync().await? {
8167            return Ok(());
8168        }
8169        // Judged by the review record itself, not by `status`: a solo
8170        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8171        // reentry (see `judge`), and trusting it here is exactly how a run
8172        // that exhausted its review budget got gated and merged a second
8173        // time around. `review_conclusion` recomputes the review loop's own
8174        // verdict from the round records themselves — `Gating` for a clean
8175        // round or a hand-off (see `stop_reviewing`), anything else means the
8176        // loop is still going or genuinely blocked.
8177        // A base the winner could not be replayed onto is a decision, not a
8178        // round: there is no landing tree to gate. Read as its own record for
8179        // the same reason the review verdict is.
8180        if self.state.status == RunStatus::Failed
8181            || self
8182                .state
8183                .base_sync
8184                .as_ref()
8185                .is_some_and(|s| s.conflict.is_some())
8186            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8187                != Some(RunStatus::Gating)
8188        {
8189            return Ok(());
8190        }
8191        let Some(winner) = self.state.winner().cloned() else {
8192            return Ok(());
8193        };
8194        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8195        self.state.reconcile_verification_head(&head);
8196        self.state.save()?;
8197        let wait = self.repository_wait();
8198        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8199            return Ok(());
8200        }
8201        if self.state.gate_ran {
8202            // `review_loop` derives its conclusion from the clean review
8203            // record on every reentry and therefore puts a completed run back
8204            // in `Gating`. A recorded gate is a stronger, terminal fact:
8205            // retain its original command output (or lack of any, for a repo
8206            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8207            // doc) and restore `Blocked` on a real failure rather than
8208            // pretending the command is still running or running it a second
8209            // time. `gate_ran == false` remains the only shape — unattempted,
8210            // or a resource-blocked retry — that may still need to execute a
8211            // command.
8212            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8213                self.state.status = RunStatus::Blocked;
8214                self.state.save()?;
8215            }
8216            return Ok(());
8217        }
8218        self.state.status = RunStatus::Gating;
8219        let mut outcomes = self.run_gate(&winner).await?;
8220        let mut attempted_head = self.state.gate_head.clone();
8221        loop {
8222            // A resource-blocked outcome means the gate command never actually
8223            // ran - the shared build cache could not be acquired or confirmed
8224            // fresh in time - which is evidence about the machine, not about
8225            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8226            // Recording it as a red gate would mark a run `Blocked` on nothing
8227            // but contention magi has already logged; leaving `self.state.gate`
8228            // empty and `self.state.gate_ran` false instead keeps the shape
8229            // this function already treats as "still needs to run" (see the
8230            // early-return above), so the next call retries the command
8231            // rather than concluding anything.
8232            if verify_inconclusive(&outcomes) {
8233                self.state.save()?;
8234                return Ok(());
8235            }
8236            if outcomes.iter().all(CommandOutcome::ok) {
8237                break;
8238            }
8239            let fixes_before = self.state.gate_fixes.len();
8240            match self.gate_fix_round(&winner, &outcomes).await? {
8241                GateFix::Retry => {
8242                    outcomes = self.run_gate(&winner).await?;
8243                    attempted_head = self.state.gate_head.clone();
8244                }
8245                GateFix::Stop => {
8246                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8247                    if self.state.gate_fixes.len() > fixes_before
8248                        && attempted_head.as_deref() != Some(current.as_str())
8249                        && verify_current_head(&mut self.state, &winner, &wait).await?
8250                    {
8251                        // A new SHA owes a gate even when a spent fixer changed only
8252                        // commit metadata. This does not grant another repair round.
8253                        outcomes = self.run_gate(&winner).await?;
8254                        attempted_head = self.state.gate_head.clone();
8255                    } else {
8256                        break;
8257                    }
8258                }
8259                GateFix::Defer => {
8260                    self.state.save()?;
8261                    return Ok(());
8262                }
8263            }
8264        }
8265        let passed = outcomes.iter().all(CommandOutcome::ok);
8266        self.state.gate = outcomes;
8267        self.state.gate_ran = true;
8268        self.state.gate_head = attempted_head;
8269        if !self.state.gate.is_empty()
8270            && !verify_inconclusive(&self.state.gate)
8271            && let Some(head) = self.state.gate_head.clone()
8272        {
8273            self.state.record_base_progress(&head);
8274        }
8275        self.state.save()?;
8276        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8277            return Ok(());
8278        }
8279        if !passed {
8280            self.state.status = RunStatus::Blocked;
8281            let spent = self.state.gate_fixes.len();
8282            self.state.event(
8283                "gate",
8284                if spent == 0 {
8285                    "gate failed; not merging".to_owned()
8286                } else {
8287                    format!("gate failed after {spent} gate-fix round(s); not merging")
8288                },
8289            );
8290        }
8291        self.state.save()?;
8292        Ok(())
8293    }
8294
8295    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8296    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8297    /// after review is clean and never on a candidate awaiting judging.
8298    ///
8299    /// Never fails the run: a non-zero exit or timeout is a warning and a
8300    /// recorded outcome, and the gate remains the single arbiter. Nothing
8301    /// configured means nothing happens - no event, no commit. `commit_all`
8302    /// commits any leftover change under the neutral identity and returns
8303    /// `false` when the tree is clean, so no empty commit is ever made.
8304    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8305        let commands = state.config.verify.pre_gate.clone();
8306        if commands.is_empty() {
8307            return;
8308        }
8309        let shell = state.config.shell();
8310        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8311        let (outcomes, _) = run_commands(
8312            state,
8313            "pre_gate",
8314            "pre_gate",
8315            0,
8316            &shell,
8317            &commands,
8318            &winner.worktree,
8319            timeout,
8320        )
8321        .await;
8322        for o in &outcomes {
8323            if !o.ok() {
8324                tracing::warn!(
8325                    "pre_gate `{}` failed ({:?}); the gate decides",
8326                    o.command,
8327                    o.code
8328                );
8329            }
8330            state.event(
8331                "pre_gate",
8332                format!(
8333                    "`{}` -> {}",
8334                    o.command,
8335                    if o.ok() {
8336                        "pass".to_owned()
8337                    } else {
8338                        format!(
8339                            "FAIL ({:?})\n{}",
8340                            o.code,
8341                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8342                        )
8343                    }
8344                ),
8345            );
8346        }
8347        state.pre_gate = outcomes;
8348        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8349            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8350                Ok(head) => {
8351                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8352                    state.pre_gate_commit = Some(head);
8353                }
8354                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8355            },
8356            Ok(false) => {}
8357            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8358        }
8359        if let Err(e) = state.save() {
8360            tracing::warn!("could not persist the pre_gate record: {e:#}");
8361        }
8362    }
8363
8364    /// Run `verify.gate` once against the winner's current tree, logging one
8365    /// event per command. Empty when nothing is configured.
8366    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8367        let wait = self.repository_wait();
8368        run_landing_gate(&mut self.state, winner, &wait).await
8369    }
8370
8371    /// One bounded fix round for a failing gate.
8372    ///
8373    /// The fixer is told the failure came from the gate itself, not from a
8374    /// reviewer, and is shown the failed commands, their exit codes and a tail
8375    /// of their output - whatever `[verify].gate` holds, nothing here knows
8376    /// what those commands run. Only a normal non-zero exit that printed
8377    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8378    /// command or a full disk says nothing about the code, and a fixer sent
8379    /// after it can only appease the machine. The round is judged by what git
8380    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8381    /// again before the gate does, so a fix cannot trade a green gate for a
8382    /// red e2e unnoticed.
8383    async fn gate_fix_round(
8384        &mut self,
8385        winner: &Candidate,
8386        outcomes: &[CommandOutcome],
8387    ) -> Result<GateFix> {
8388        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8389            g.phase == crate::repair_budget::Phase::Fixing
8390                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8391        }) {
8392            self.state.gate_fixes.len().saturating_add(1)
8393        } else {
8394            self.state.config.graph.gate_fix_rounds
8395        };
8396        let spent = self.state.gate_fixes.len();
8397        if spent >= cap {
8398            if cap > 0 {
8399                self.state.event(
8400                    "gate",
8401                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8402                );
8403            }
8404            return Ok(GateFix::Stop);
8405        }
8406        if !gate_fixable(outcomes) {
8407            self.state.event(
8408                "gate",
8409                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8410                 command or similar); not spending a fix round on it",
8411            );
8412            return Ok(GateFix::Stop);
8413        }
8414        let min_free = self.state.config.disk.min_free_bytes;
8415        if min_free > 0 {
8416            match crate::disk::free_bytes(&winner.worktree) {
8417                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8418                Ok(free) => {
8419                    self.state.event(
8420                        "gate",
8421                        format!(
8422                            "only {free} bytes free ({min_free} required by `[disk] \
8423                             min_free_bytes`); not spending a fix round on a failure the disk \
8424                             may explain"
8425                        ),
8426                    );
8427                    return Ok(GateFix::Stop);
8428                }
8429                Err(e) => {
8430                    self.state.event(
8431                        "gate",
8432                        format!("free disk space could not be measured ({e:#}); no fix round"),
8433                    );
8434                    return Ok(GateFix::Stop);
8435                }
8436            }
8437        }
8438
8439        let attempt = spent + 1;
8440        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8441        let base = self.landing_base();
8442        let fix_prompt = prompt::gate_fix_in(
8443            &self.state.instruction,
8444            &failed,
8445            attempt,
8446            cap,
8447            &self.state.config.graph.language,
8448            &self.state.config.graph.github_language,
8449        );
8450        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8451        let sessions = self.state.config.graph.sessions;
8452        let artifacts = self.merge_repair_artifacts();
8453        self.state.event(
8454            "gate",
8455            format!("gate failed; gate-fix round {attempt} of {cap}"),
8456        );
8457        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8458        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8459        // Persist before invoking the fixer: it may commit and then be
8460        // interrupted before its result (or the following e2e) is saved.
8461        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8462            head: before.clone(),
8463            passed: false,
8464            outcomes: Vec::new(),
8465        });
8466        self.state.save()?;
8467        let (_, seat, out) = self
8468            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8469                prompt: fix_prompt.clone(),
8470                spec,
8471                seat,
8472                cwd: winner.worktree.clone(),
8473                timeout,
8474                allow_write: true,
8475                sessions,
8476                artifacts: artifacts.clone(),
8477                stem: format!("gate-fix-{attempt}"),
8478                handover: None,
8479            })
8480            .await;
8481        // The fixer was never asked: no round is spent (nothing is recorded)
8482        // and a later reentry asks the same agent again.
8483        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8484            return Ok(GateFix::Defer);
8485        }
8486        let mut record = GateFixRecord {
8487            agent: seat.agent.clone(),
8488            failed,
8489            notes: String::new(),
8490            committed: false,
8491            error: None,
8492        };
8493        match out {
8494            AgentOutcome::Ok(o) => {
8495                // A missing report is not a failed fix: the round is judged
8496                // by the tree below, and the report only carries prose.
8497                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8498                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8499                }
8500            }
8501            AgentOutcome::Dropped(_) => {
8502                record.error = Some("the CLI dropped the stream".to_owned());
8503            }
8504            AgentOutcome::Quota(o) => {
8505                self.state.quota.push(QuotaLoss::from_report(
8506                    seat.key.clone(),
8507                    "gate-fix",
8508                    o.quota.as_ref(),
8509                ));
8510                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8511            }
8512            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8513        }
8514        self.state.seats.insert(seat.key.clone(), seat);
8515        let coordination = self.coordinate(&winner.worktree).await?;
8516        if let Ok(r) = git::rescue_commit(
8517            &winner.worktree,
8518            &format!("magi: gate fix {attempt} (uncommitted work)"),
8519        )
8520        .await
8521        {
8522            self.state.note_withheld("gate-fix", &r.withheld);
8523        }
8524        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8525        record.committed = after != before;
8526        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8527        let note = record.error.clone();
8528        self.state.gate_fixes.push(record);
8529        self.state.save()?;
8530        if !changed {
8531            self.state.event(
8532                "gate",
8533                match note {
8534                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8535                    None => format!("gate-fix round {attempt}: the tree did not change"),
8536                },
8537            );
8538            return Ok(GateFix::Stop);
8539        }
8540        self.state.event(
8541            "gate",
8542            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8543        );
8544
8545        drop(coordination);
8546        let wait = self.repository_wait();
8547        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8548    }
8549
8550    // ---------------------------------------------------------------- merge
8551
8552    /// One read-only rewrite by the summary's author; a text the gate still
8553    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8554    /// applies on their word or on silence. `None` means the run is parked on
8555    /// that question and nothing may be pushed or opened yet.
8556    async fn guarded_pr_message(
8557        &mut self,
8558        winner: &Candidate,
8559        facts: Option<&BranchFacts>,
8560        posting: bool,
8561    ) -> Result<Option<PrMessage>> {
8562        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8563        let head = match final_head.as_deref() {
8564            Some(h) => crate::pr_outcome::FinalHead::At(h),
8565            None => crate::pr_outcome::FinalHead::Unknown,
8566        };
8567        // Only a pull request that is about to be posted pays for a summary.
8568        let summarized = if posting {
8569            Some(self.summarize_change(winner, final_head.as_deref()).await)
8570        } else {
8571            None
8572        };
8573        // A busy shared resource is not a permanent cause: publishing an
8574        // "unavailable" description would stick (a later head move never
8575        // edits the body). Stop before anything is pushed or opened; the
8576        // run is `Blocked` and a resume asks the summary again.
8577        if self.state.resource_block.is_some() {
8578            self.state.event(
8579                "pr-summary",
8580                "a shared resource was busy; nothing was published, resume to summarize again",
8581            );
8582            return Ok(None);
8583        }
8584        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8585            anyhow::bail!(
8586                "the final commit moved during public description generation; retry before publishing"
8587            );
8588        }
8589        let note = match (&summarized, &self.state.change_summary) {
8590            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8591            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8592            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8593            (None, _) => ChangeNote::NotAsked,
8594        };
8595        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8596        if !posting {
8597            let identity = crate::scrub::Identity::current();
8598            return Ok(Some(PrMessage {
8599                title: crate::scrub::scrub(&pr.title, &identity),
8600                body: crate::scrub::scrub(&pr.body, &identity),
8601            }));
8602        }
8603        // This very text was already rejected and asked about: no second
8604        // judge call, rewrite or question, just carry the record on.
8605        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8606        if let Some(known) = self
8607            .state
8608            .github_text
8609            .clone()
8610            .filter(|g| g.fingerprint == raw_fp)
8611        {
8612            return self.settle_withheld(winner, &pr, known).await;
8613        }
8614        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8615        let mut violations =
8616            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8617        let identity = crate::scrub::Identity::current();
8618        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8619            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8620            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8621        {
8622            violations.push(crate::github_text::Violation::SensitiveData);
8623        }
8624        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8625        // only a language violation starts one; sensitive data alone goes
8626        // straight to the owner, with the redacted text as the fallback.
8627        let has_language = violations
8628            .iter()
8629            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8630        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8631            self.state.event(
8632                "github-text",
8633                if has_language {
8634                    format!("description rejected: {violations:?}; requesting one rewrite")
8635                } else {
8636                    format!("description rejected: {violations:?}; asking the owner")
8637                },
8638            );
8639            let mut rewritten = false;
8640            let mut rejected_rewrite = None;
8641            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8642            {
8643                let output = text;
8644                if let Some(title) = summary_title(&output) {
8645                    // Judge the redacted candidate that will actually be posted.
8646                    // Sensitive data alone must not discard a useful rewrite.
8647                    let raw_body = summary_without_title(&output);
8648                    let clean_title = crate::scrub::scrub(&title, &identity);
8649                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8650                    let redacted = clean_title != title || body != raw_body;
8651                    let title = clean_title;
8652                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8653                    if !complete {
8654                        self.state.event(
8655                            "github-text",
8656                            "description rewrite omitted required findings",
8657                        );
8658                    }
8659                    if !body.trim().is_empty() {
8660                        body.push_str("\n\n");
8661                        body.push_str(&public_pr_evidence(&self.state, head));
8662                    }
8663                    let marker = format!("magi:run/{}", self.state.id);
8664                    if !body.trim().is_empty() && !body.contains(&marker) {
8665                        body.push_str(&format!("\n\n{marker}"));
8666                    }
8667                    let rewrite = PrMessage {
8668                        title: title.clone(),
8669                        body: body.clone(),
8670                    };
8671                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8672                    let mut failed = crate::github_text::check_for(
8673                        &self.state.config.graph,
8674                        &title,
8675                        &body,
8676                        redecision,
8677                    );
8678                    if (!crate::github_text::shareable(&title)
8679                        || !crate::github_text::shareable(&body))
8680                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8681                    {
8682                        failed.push(crate::github_text::Violation::SensitiveData);
8683                    }
8684                    if complete
8685                        && !title.trim().is_empty()
8686                        && !body.trim().is_empty()
8687                        && !failed.is_empty()
8688                    {
8689                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8690                            title: title.clone(),
8691                            body: body.clone(),
8692                            failed: failed
8693                                .iter()
8694                                .map(|v| crate::github_text::category(*v).to_owned())
8695                                .collect(),
8696                        });
8697                    }
8698                    if complete
8699                        && !title.trim().is_empty()
8700                        && !raw_body.trim().is_empty()
8701                        && failed.is_empty()
8702                    {
8703                        decision = redecision;
8704                        pr = PrMessage { title, body };
8705                        rewritten = true;
8706                        self.state
8707                            .event("github-text", "description rewrite passed");
8708                        if redacted {
8709                            self.state.event(
8710                                "github-text",
8711                                "sensitive data removed from description rewrite",
8712                            );
8713                        }
8714                    }
8715                }
8716            }
8717            if !rewritten {
8718                self.state.event(
8719                    "github-text",
8720                    "description rewrite unavailable or rejected; asking the owner",
8721                );
8722                if let Some(w) =
8723                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8724                {
8725                    // A different text than the one asked about before: that
8726                    // question no longer describes anything.
8727                    self.retire_github_text_question("the withheld text changed");
8728                    let record = crate::run::GithubTextAsk {
8729                        rewrite: rejected_rewrite,
8730                        fingerprint: raw_fp,
8731                        title: w.title,
8732                        body: w.body,
8733                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8734                        question: None,
8735                        asks: 0,
8736                        resolved: false,
8737                        chosen_title: None,
8738                        choice: None,
8739                    };
8740                    return self.settle_withheld(winner, &pr, record).await;
8741                }
8742            }
8743        }
8744        let (title, body) =
8745            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8746        Ok(Some(PrMessage { title, body }))
8747    }
8748
8749    /// Describe the whole change from its final diff in either run mode.
8750    ///
8751    /// The summary is tied to the final commit and to the merge base of the
8752    /// resolved remote base; a record for any other pair is dropped and the
8753    /// agent asked again, so a description never outlives the diff it was
8754    /// written from. The seat is read-only and the prompt tells it to read the
8755    /// diff and the code around it. `[roles] synthesizer` selects the agents
8756    /// (unset: the default pick), each asked at most once, advancing on an
8757    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8758    /// carries the reason it is unavailable; nothing is invented then.
8759    async fn summarize_change(
8760        &mut self,
8761        winner: &Candidate,
8762        head: Option<&str>,
8763    ) -> std::result::Result<(), String> {
8764        let Some(head) = head else {
8765            return Err("the final commit could not be read".to_owned());
8766        };
8767        let remote = self.state.config.merge.remote.clone();
8768        let tracking = format!("{remote}/{}", self.state.base_branch);
8769        if !git::rev_exists(&winner.worktree, &tracking).await {
8770            return Err(format!("the remote base `{tracking}` could not be read"));
8771        }
8772        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8773            Ok(b) if !b.is_empty() => b,
8774            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8775        };
8776        let evidence = crate::pr_outcome::summary_evidence(
8777            &self.state,
8778            crate::pr_outcome::FinalHead::At(head),
8779        );
8780        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8781        if self
8782            .state
8783            .change_summary
8784            .as_ref()
8785            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8786        {
8787            return Ok(());
8788        }
8789        if self.state.change_summary.take().is_some() {
8790            self.state.event(
8791                "pr-summary",
8792                "the final diff or evidence changed; the earlier change summary was discarded",
8793            );
8794        }
8795        // Save the source snapshot even if no generation agent is available.
8796        if let Err(e) = write_artifact(
8797            &self.state,
8798            &format!("public-description-{fingerprint}.md"),
8799            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8800        ) {
8801            tracing::warn!("could not save public description evidence: {e:#}");
8802        }
8803        let chain = match agent::pick_chain(
8804            &self.state.config.agents,
8805            self.state.config.roles.synthesizer.as_ref(),
8806            &agent::installed,
8807            "synthesizer",
8808        ) {
8809            Ok(chain) => chain,
8810            Err(e) => {
8811                tracing::warn!("no agent for the change summary: {e:#}");
8812                self.state.event("pr-summary", "no summary agent available");
8813                return Err("no agent was available to summarize it".to_owned());
8814            }
8815        };
8816        let subjects = git::subjects(&winner.worktree, &base, head)
8817            .await
8818            .unwrap_or_default();
8819        let stat = git::diff_stat(&winner.worktree, &base, head)
8820            .await
8821            .unwrap_or_default();
8822        let stat: String = stat.chars().take(6_000).collect();
8823        let mut prompt = crate::prompt::change_summary_in(
8824            &base,
8825            head,
8826            &subjects,
8827            stat.trim(),
8828            &self.state.config.graph.github_language,
8829        );
8830        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8831        let sessions = self.state.config.graph.sessions;
8832        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8833        let prompts = self.state.config.prompts.clone();
8834        let cache = self.state.config.cache_dir();
8835        let run = self.state.id.clone();
8836        for (i, spec) in chain.into_iter().enumerate() {
8837            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8838            let github_spec = match agent::for_github_text(
8839                &spec,
8840                &self.state.config.graph.github_language,
8841                &winner.worktree,
8842            ) {
8843                Ok(spec) => spec,
8844                Err(e) => {
8845                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8846                    continue;
8847                }
8848            };
8849            let job = SeatJob {
8850                spec: github_spec,
8851                seat,
8852                cwd: winner.worktree.clone(),
8853                prompt: prompt.clone(),
8854                timeout,
8855                allow_write: false,
8856                sessions,
8857                artifacts: agent::artifacts_dir(&self.state.dir()),
8858                stem: if i == 0 {
8859                    "change-summary".to_owned()
8860                } else {
8861                    format!("change-summary-{}", spec.id)
8862                },
8863                handover: None,
8864            };
8865            let ctx = WaveCtx {
8866                run: &run,
8867                node: "pr-summary",
8868                prompts: &prompts,
8869                cache: cache.as_deref(),
8870                round: None,
8871                carry_seats: false,
8872                pauses: Some((&self.pause, &self.interrupt)),
8873            };
8874            let (seat, outcome) =
8875                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8876            self.state.seats.insert(seat.key.clone(), seat);
8877            let text = match outcome {
8878                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8879                AgentOutcome::ResourceBlocked(why) => {
8880                    // No agent was asked: the run stops resumably before
8881                    // anything is published (see `guarded_pr_message`).
8882                    if self.state.resource_block.is_none() {
8883                        self.state.resource_block = Some(crate::run::ResourceBlock {
8884                            cause: crate::run::ResourceBlockCause::Other,
8885                            node: "pr-summary".to_owned(),
8886                            kind: "shared resource".to_owned(),
8887                            detail: why,
8888                            at: Timestamp::now(),
8889                        });
8890                    }
8891                    return Err("a shared resource was busy".to_owned());
8892                }
8893                _ => continue,
8894            };
8895            let blind = &self.state.config.blind;
8896            let Some(title) = summary_title(&text) else {
8897                continue;
8898            };
8899            let body: String = summary_without_title(&text)
8900                .lines()
8901                .filter(|l| !l.contains("magi:run/"))
8902                .collect::<Vec<_>>()
8903                .join("\n");
8904            let body = blind::sanitize_prose(body.trim(), blind);
8905            let title = blind::sanitize_prose(&title, blind);
8906            if body.trim().is_empty() || title.trim().is_empty() {
8907                continue;
8908            }
8909            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8910                continue;
8911            }
8912            // The branch must still be the one the agent was shown.
8913            if git::rev_parse(&winner.worktree, "HEAD")
8914                .await
8915                .ok()
8916                .as_deref()
8917                != Some(head)
8918            {
8919                return Err("the branch moved while the summary was written".to_owned());
8920            }
8921            if git::merge_base(&winner.worktree, &tracking, head)
8922                .await
8923                .ok()
8924                .as_deref()
8925                != Some(base.as_str())
8926                || crate::pr_outcome::summary_evidence(
8927                    &self.state,
8928                    crate::pr_outcome::FinalHead::At(head),
8929                ) != evidence
8930            {
8931                return Err("the summary evidence changed while it was written".to_owned());
8932            }
8933            self.state.change_summary = Some(crate::run::ChangeSummary {
8934                head: head.to_owned(),
8935                base,
8936                evidence: fingerprint,
8937                title,
8938                body,
8939            });
8940            self.state.event(
8941                "pr-summary",
8942                format!("change summary written by {} for the final diff", spec.id),
8943            );
8944            if let Err(e) = self.state.save() {
8945                tracing::warn!("could not save the change summary: {e:#}");
8946            }
8947            return Ok(());
8948        }
8949        self.state
8950            .event("pr-summary", "no agent produced a change summary");
8951        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8952    }
8953
8954    /// One read-only rewrite request for a withheld description. The
8955    /// implementer's own conversation is tried first when it exists and can be
8956    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8957    /// writes it in a separate conversation, skipping the implementer's agent.
8958    /// The prompt carries the whole text, so no agent needs any memory. Each
8959    /// agent is asked at most once and the chain advances on an error, quota,
8960    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8961    /// that has run out of quota must not end the rewrite. Nothing is recorded
8962    /// as a handover.
8963    async fn rewrite_pr_text(
8964        &mut self,
8965        winner: &Candidate,
8966        pr: &PrMessage,
8967        violations: &[crate::github_text::Violation],
8968    ) -> Option<String> {
8969        let prompt = format!(
8970            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8971            pr.title, pr.body
8972        );
8973        let sessions = self.state.config.graph.sessions;
8974        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8975        // (spec, the resumed implementer's seat). A fallback seat is created
8976        // only when its turn comes, so an unused fallback leaves no record.
8977        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8978        let key = format!("impl-{}", winner.label);
8979        if let Some(spec) = self
8980            .state
8981            .config
8982            .agents
8983            .iter()
8984            .find(|a| a.id == winner.agent)
8985            .cloned()
8986            && let Some(existing) = self.state.seats.get(&key)
8987            && existing.agent == spec.id
8988            && agent::has_session(spec.kind, existing, sessions)
8989        {
8990            attempts.push((spec, Some(existing.clone())));
8991        }
8992        match agent::pick_chain(
8993            &self.state.config.agents,
8994            self.state.config.roles.language_judge.as_ref(),
8995            &agent::installed,
8996            "language_judge",
8997        ) {
8998            Ok(chain) => {
8999                for spec in chain {
9000                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9001                        continue;
9002                    }
9003                    attempts.push((spec, None));
9004                }
9005            }
9006            Err(e) if attempts.is_empty() => {
9007                tracing::warn!("no agent for the description rewrite: {e:#}");
9008                self.state
9009                    .event("github-text", "no rewrite agent available");
9010                return None;
9011            }
9012            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9013        }
9014        let prompts = self.state.config.prompts.clone();
9015        let cache = self.state.config.cache_dir();
9016        let run = self.state.id.clone();
9017        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9018            let resumed = resumed_seat.is_some();
9019            let seat = resumed_seat.unwrap_or_else(|| {
9020                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9021            });
9022            let github_spec = match agent::for_github_text(
9023                &spec,
9024                &self.state.config.graph.github_language,
9025                &winner.worktree,
9026            ) {
9027                Ok(spec) => spec,
9028                Err(e) => {
9029                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9030                    continue;
9031                }
9032            };
9033            let job = SeatJob {
9034                spec: github_spec,
9035                seat,
9036                cwd: winner.worktree.clone(),
9037                prompt: prompt.clone(),
9038                timeout: retry_budget(timeout_full, resumed),
9039                allow_write: false,
9040                sessions,
9041                artifacts: agent::artifacts_dir(&self.state.dir()),
9042                stem: if i == 0 {
9043                    "github-text-rewrite".to_owned()
9044                } else {
9045                    format!("github-text-rewrite-{}", spec.id)
9046                },
9047                handover: None,
9048            };
9049            let ctx = WaveCtx {
9050                run: &run,
9051                node: "github-text",
9052                prompts: &prompts,
9053                cache: cache.as_deref(),
9054                round: None,
9055                carry_seats: false,
9056                pauses: Some((&self.pause, &self.interrupt)),
9057            };
9058            let (seat, outcome) =
9059                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9060            self.state.seats.insert(seat.key.clone(), seat);
9061            match outcome {
9062                AgentOutcome::Ok(o)
9063                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9064                {
9065                    return Some(o.text);
9066                }
9067                AgentOutcome::ResourceBlocked(_) => return None,
9068                _ => {}
9069            }
9070        }
9071        None
9072    }
9073
9074    /// Abandon the standing posting-gate question, if any.
9075    fn retire_github_text_question(&mut self, why: &str) {
9076        let Some(id) = self
9077            .state
9078            .github_text
9079            .as_ref()
9080            .and_then(|g| g.question.clone())
9081        else {
9082            return;
9083        };
9084        let _ = ask::Questions::open().update(&id, |q| {
9085            q.abandon(why);
9086            Ok(())
9087        });
9088    }
9089
9090    /// Carry a withheld text to its end: file the question, park while it is
9091    /// open, and on an answer, silence or timeout pick the text to post. The
9092    /// decision is saved before it is used, so it applies once.
9093    async fn settle_withheld(
9094        &mut self,
9095        winner: &Candidate,
9096        pr: &PrMessage,
9097        mut g: crate::run::GithubTextAsk,
9098    ) -> Result<Option<PrMessage>> {
9099        use crate::github_text as gt;
9100        let store = ask::Questions::open();
9101        let timeout = self.state.config.graph.answer_timeout;
9102        if !g.resolved {
9103            let mut standing = g
9104                .question
9105                .as_ref()
9106                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9107            if let Some(q) = standing.as_ref()
9108                && q.status.open()
9109                && gt::expired(q, timeout)
9110            {
9111                let why = format!("no answer within {}s of asking", timeout.max(1));
9112                standing = store
9113                    .update(&q.id, |q| {
9114                        q.abandon(&why);
9115                        Ok(())
9116                    })
9117                    .ok()
9118                    .map(|(q, ())| q);
9119            }
9120            let mut ask_again: Option<Option<Vec<String>>> = None;
9121            match standing {
9122                // Recorded but never written (a stop between the two saves),
9123                // or removed: ask again without spending the retry.
9124                None if g.asks <= 1 => {
9125                    g.asks = 0;
9126                    ask_again = Some(None);
9127                }
9128                None => {
9129                    g.resolved = true;
9130                    g.chosen_title = None;
9131                    g.choice = Some(GithubTextChoice::Fallback);
9132                }
9133                Some(q) if q.status.open() => {
9134                    self.park_on_github_text(&g)?;
9135                    return Ok(None);
9136                }
9137                Some(q) => match gt::read_reply(&q) {
9138                    gt::Reply::Fallback => {
9139                        g.resolved = true;
9140                        g.chosen_title = None;
9141                        g.choice = Some(GithubTextChoice::Fallback);
9142                    }
9143                    gt::Reply::Original => {
9144                        g.resolved = true;
9145                        g.chosen_title = None;
9146                        g.choice = Some(GithubTextChoice::Original);
9147                    }
9148                    gt::Reply::Rewrite => {
9149                        g.resolved = true;
9150                        g.chosen_title = None;
9151                        g.choice = Some(if g.rewrite.is_some() {
9152                            GithubTextChoice::Rewrite
9153                        } else {
9154                            GithubTextChoice::Fallback
9155                        });
9156                    }
9157                    gt::Reply::Title(text) => {
9158                        let first = text
9159                            .lines()
9160                            .map(str::trim)
9161                            .find(|l| !l.is_empty())
9162                            .unwrap_or("");
9163                        let decision = if first.is_empty() {
9164                            None
9165                        } else {
9166                            let candidate = PrMessage {
9167                                title: first.to_owned(),
9168                                body: String::new(),
9169                            };
9170                            self.judge_pr_language(&winner.worktree, &candidate).await
9171                        };
9172                        match gt::vet_title(
9173                            &text,
9174                            decision,
9175                            self.state.config.graph.github_english(),
9176                        ) {
9177                            Ok(title) => {
9178                                g.resolved = true;
9179                                g.chosen_title = Some(title);
9180                                g.choice = Some(GithubTextChoice::MyTitle);
9181                            }
9182                            Err(cats) if g.asks < 2 => {
9183                                ask_again = Some(Some(
9184                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9185                                ))
9186                            }
9187                            Err(_) => {
9188                                g.resolved = true;
9189                                g.chosen_title = None;
9190                                g.choice = Some(GithubTextChoice::Fallback);
9191                            }
9192                        }
9193                    }
9194                },
9195            }
9196            if let Some(rejected) = ask_again {
9197                // A retry (after a rejected replacement title) is still about
9198                // the original text: the same ORIGINAL / FALLBACK comparison.
9199                let w = gt::Withheld {
9200                    title: g.title,
9201                    body: g.body,
9202                    categories: g.categories.clone(),
9203                };
9204                // Named by the original text's fingerprint alone, so a resume
9205                // rewrites the same file and the detail stays identical.
9206                let artifact = match crate::run::write_artifact(
9207                    &self.state,
9208                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9209                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9210                ) {
9211                    Ok(p) => Some(p),
9212                    Err(e) => {
9213                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9214                        None
9215                    }
9216                };
9217                let (detail, comparison) = gt::question_content(
9218                    &w,
9219                    &pr.title,
9220                    &pr.body,
9221                    &self.state.id,
9222                    rejected.as_deref(),
9223                    artifact.as_deref(),
9224                    g.rewrite.as_ref(),
9225                );
9226                let mut q = ask::Question::new(
9227                    self.state.id.clone(),
9228                    gt::ASK_NODE.to_owned(),
9229                    gt::ASK_SEAT.to_owned(),
9230                    gt::question_summary(&self.state.config.graph.language, &w),
9231                    detail,
9232                    gt::question_choices(&w, g.rewrite.as_ref()),
9233                );
9234                q.posting_comparison = Some(comparison);
9235                q.answer_timeout = timeout;
9236                // Recorded before it is written: a stop in between leaves a
9237                // record naming a question the resume then files again.
9238                g.question = Some(q.id.clone());
9239                g.asks += 1;
9240                self.state.github_text = Some(g.clone());
9241                self.state.event(
9242                    "github-text",
9243                    format!("asking the owner about the withheld text ({})", q.short()),
9244                );
9245                self.state.save()?;
9246                store
9247                    .put_executable(&mut q)
9248                    .context("file the posting-gate question")?;
9249                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9250                    tracing::warn!(
9251                        "could not notify about posting-gate question {}: {e:#}",
9252                        q.short()
9253                    );
9254                }
9255                self.park_on_github_text(&g)?;
9256                return Ok(None);
9257            }
9258        }
9259        let resolved = g.chosen_title.clone();
9260        let original = g.choice == Some(GithubTextChoice::Original);
9261        // Field names only: the event log never records the text.
9262        self.state.event(
9263            "github-text",
9264            if original {
9265                format!(
9266                    "posting the original text for: {}",
9267                    match (g.title, g.body) {
9268                        (true, true) => "title, description",
9269                        (true, false) => "title",
9270                        _ => "description",
9271                    }
9272                )
9273            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9274                "posting the saved rewrite title and description".to_owned()
9275            } else if resolved.is_some() {
9276                "posting the owner's replacement title".to_owned()
9277            } else {
9278                "posting the neutral text for the withheld fields".to_owned()
9279            },
9280        );
9281        self.state.github_text = Some(g.clone());
9282        self.state.save()?;
9283        if g.choice == Some(GithubTextChoice::Rewrite)
9284            && let Some(r) = g.rewrite.as_ref()
9285        {
9286            return Ok(Some(PrMessage {
9287                title: r.title.clone(),
9288                body: r.body.clone(),
9289            }));
9290        }
9291        let w = gt::Withheld {
9292            title: g.title,
9293            body: g.body,
9294            categories: g.categories.clone(),
9295        };
9296        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9297        if let Some(t) = resolved {
9298            title = t;
9299        }
9300        // Every withheld field is now neutral or vetted English.
9301        let english = gt::LanguageDecision {
9302            title_english: true,
9303            body_english: true,
9304            body_complete: true,
9305        };
9306        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9307        if original {
9308            // The owner's explicit choice overrides the gate for the fields it
9309            // was made for; the others keep what the gate produced.
9310            if g.title {
9311                title = pr.title.clone();
9312            }
9313            if g.body {
9314                body = pr.body.clone();
9315            }
9316        }
9317        Ok(Some(PrMessage { title, body }))
9318    }
9319
9320    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9321        self.state.github_text = Some(g.clone());
9322        self.state.end_pass_parked(ParkCause::Question);
9323        self.state.parked = true;
9324        self.state.event(
9325            "github-text",
9326            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9327        );
9328        self.state.save()
9329    }
9330
9331    /// Ask `[roles] language_judge` about this text and record which source
9332    /// decided; the text itself is never recorded.
9333    async fn judge_pr_language(
9334        &mut self,
9335        cwd: &Path,
9336        pr: &PrMessage,
9337    ) -> Option<crate::github_text::LanguageDecision> {
9338        self.state.config.roles.language_judge.as_ref()?;
9339        if !self.state.config.graph.language_gate() {
9340            return None;
9341        }
9342        let decision =
9343            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9344        self.state.event(
9345            "github-text",
9346            if decision.is_some() {
9347                "language decided by the language judge"
9348            } else {
9349                "language judge unavailable; using built-in heuristics"
9350            },
9351        );
9352        decision
9353    }
9354
9355    async fn merge(&mut self) -> Result<()> {
9356        if self.resource_halt()? || self.hold_base_sync().await? {
9357            return Ok(());
9358        }
9359        // Same reasoning as `gate`: ask the review and gate records directly
9360        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9361        // skip can rewrite on reentry to something that no longer says
9362        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9363        // so a hand-off (open findings, green verification) reaches merge
9364        // exactly like a genuinely clean round does.
9365        //
9366        // A run resumed mid-`land` never reaches here at all: `execute`
9367        // recognises `RunStatus::Landing` before it even calls `prep`, and
9368        // routes straight to `run_land` instead. That has to happen a level
9369        // up from this function, not with a check in here, because
9370        // `review_loop`'s own status recomputation (see its doc) runs
9371        // *before* `merge` on every reentry and would otherwise overwrite
9372        // the `Landing` marker with `Gating` before this node ever saw it.
9373        if self
9374            .state
9375            .base_sync
9376            .as_ref()
9377            .is_some_and(|s| s.conflict.is_some())
9378            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9379                != Some(RunStatus::Gating)
9380            // `gate_ran == false` is not "passed" - `gate` leaves it false
9381            // both before it has ever run and when its last attempt was
9382            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9383            // permission to merge on nothing but the review record. Only a
9384            // gate that actually ran may proceed; an intentional local
9385            // review can also finish with verification explicitly not configured.
9386            // `RunState::gate_status` computes that distinction.
9387            || !self.state.gate_allows_progress()
9388        {
9389            return Ok(());
9390        }
9391        // This node's own record, not `status`: `status == Ready` is not
9392        // unique to the harmless `MergeMode::None` path this line was
9393        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9394        // run's PR was closed without merging — and on that run `mode` is
9395        // still `Pr`, so a reentry that fell through here would push and
9396        // open a second pull request. `self.state.merge` is set exactly once
9397        // this node (or `land`) has already produced a verdict, under every
9398        // mode, which is what "already done" actually means here.
9399        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9400            return Ok(());
9401        }
9402        let Some(winner) = self.state.winner().cloned() else {
9403            return Ok(());
9404        };
9405        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9406        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9407            self.state.status = RunStatus::Blocked;
9408            self.state.event(
9409                "gate",
9410                "winner changed after gate; fresh verification required",
9411            );
9412            self.state.save()?;
9413            return Ok(());
9414        }
9415        let wait = self.repository_wait();
9416        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9417            return Ok(());
9418        }
9419        let repo = self.state.repo.clone();
9420        let base = self.state.base_branch.clone();
9421        let mode = self.state.config.merge.mode;
9422        let style = self.state.config.merge.style;
9423        let facts = if is_review_run(&self.state) {
9424            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9425            let start = review_base(
9426                &repo,
9427                &self.state.config.merge.remote,
9428                &base,
9429                &self.state.base_commit,
9430                &winner.branch,
9431            )
9432            .await;
9433            branch_facts(&repo, &start, &winner.branch).await
9434        } else {
9435            None
9436        };
9437        // `None` when the base could not be freshly read: then an adopted
9438        // pull request's title is left alone.
9439        let leaked = if is_review_run(&self.state) {
9440            leaked_subjects(&self.state, &winner.branch).await
9441        } else {
9442            Some(Vec::new())
9443        };
9444        // `None`: parked on the owner's word about a withheld title or body.
9445        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9446        // the resume arrives here again.
9447        let Some(pr) = self
9448            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9449            .await?
9450        else {
9451            return Ok(());
9452        };
9453        let message = pr.commit_message();
9454
9455        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9456        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9457            return Ok(());
9458        }
9459        let outcome = match mode {
9460            MergeMode::None => MergeOutcome {
9461                mode,
9462                ok: true,
9463                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9464                empty: false,
9465            },
9466            MergeMode::Pr | MergeMode::Local
9467                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9468            {
9469                MergeOutcome {
9470                    mode,
9471                    ok: false,
9472                    detail: empty_candidate_detail(&self.state, &base),
9473                    empty: true,
9474                }
9475            }
9476            MergeMode::Local => {
9477                let on = git::current_branch(&repo).await?;
9478                if on.as_deref() != Some(base.as_str()) {
9479                    MergeOutcome {
9480                        mode,
9481                        ok: false,
9482                        detail: format!(
9483                            "{} has {} checked out, not the base branch {base}",
9484                            repo.display(),
9485                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9486                        ),
9487                        empty: false,
9488                    }
9489                } else if !git::is_clean(&repo).await? {
9490                    MergeOutcome {
9491                        mode,
9492                        ok: false,
9493                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9494                        empty: false,
9495                    }
9496                } else {
9497                    let out = match style {
9498                        MergeStyle::Merge => {
9499                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9500                        }
9501                        MergeStyle::Squash => {
9502                            git::merge_squash(&repo, &winner.branch, &message).await?
9503                        }
9504                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9505                    };
9506                    MergeOutcome {
9507                        mode,
9508                        ok: out.ok(),
9509                        detail: if out.ok() { out.stdout } else { out.stderr },
9510                        empty: false,
9511                    }
9512                }
9513            }
9514            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9515                self.state.pr_publication_pending = true;
9516                self.state.save()?;
9517                let out =
9518                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9519                self.state.pr_publication_pending = out.is_err();
9520                MergeOutcome {
9521                    mode,
9522                    ok: out.is_ok(),
9523                    detail: match out {
9524                        Ok(url) => url,
9525                        Err(e) => format!("{e:#}"),
9526                    },
9527                    empty: false,
9528                }
9529            }
9530            MergeMode::Pr => {
9531                let remote = self.state.config.merge.remote.clone();
9532                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9533                let pushed = if self.state.foreign_checkout.is_some() {
9534                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9535                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9536                } else {
9537                    git::push(&winner.worktree, &remote, &winner.branch).await?
9538                };
9539                if !pushed.ok() {
9540                    MergeOutcome {
9541                        mode,
9542                        ok: false,
9543                        detail: pushed.stderr,
9544                        empty: false,
9545                    }
9546                } else {
9547                    if self.state.foreign_checkout.is_some() {
9548                        self.state.pr_pushed_head = Some(delivery_head.clone());
9549                    }
9550                    // A retry or resume of a run whose branch already has an
9551                    // open pull request adopts it rather than failing on a
9552                    // duplicate. Only this winner branch into this base:
9553                    // `branch_for` derives the name from the run id, so a
9554                    // different run's pull request never matches.
9555                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9556                    let out = match pr_merge_plan(found) {
9557                        PrPlan::Create => {
9558                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9559                                .await
9560                        }
9561                        PrPlan::Adopt { url, title } => {
9562                            self.state
9563                                .event("merge", format!("Pr: adopted open pull request {url}"));
9564                            if self.state.github_text.as_ref().is_some_and(|g| {
9565                                g.resolved
9566                                    && g.choice == Some(GithubTextChoice::Rewrite)
9567                                    && g.rewrite.is_some()
9568                            }) {
9569                                // The owner chose the entire pair, even if the
9570                                // existing title already matches or was edited.
9571                                land::set_pr_text_verbatim(
9572                                    &winner.worktree,
9573                                    &url,
9574                                    &pr.title,
9575                                    &pr.body,
9576                                )
9577                                .await
9578                                .map(|()| url)
9579                            } else {
9580                                if title != pr.title
9581                                    && (!is_review_run(&self.state)
9582                                        || leaked
9583                                            .as_deref()
9584                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9585                                    && let Err(e) =
9586                                        if self.state.github_text.as_ref().is_some_and(|g| {
9587                                            g.resolved
9588                                                && g.title
9589                                                && g.choice == Some(GithubTextChoice::Original)
9590                                        }) {
9591                                            // The owner chose to post this title unmodified.
9592                                            land::set_pr_title_verbatim(
9593                                                &winner.worktree,
9594                                                &url,
9595                                                &pr.title,
9596                                            )
9597                                            .await
9598                                        } else {
9599                                            land::set_pr_title(
9600                                                &mut self.state,
9601                                                &winner.worktree,
9602                                                &url,
9603                                                &pr.title,
9604                                            )
9605                                            .await
9606                                        }
9607                                {
9608                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9609                                    self.state
9610                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9611                                }
9612                                Ok(url)
9613                            }
9614                        }
9615                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9616                    };
9617                    match out {
9618                        Ok(url) => MergeOutcome {
9619                            mode,
9620                            ok: true,
9621                            detail: url,
9622                            empty: false,
9623                        },
9624                        Err(e) => MergeOutcome {
9625                            mode,
9626                            ok: false,
9627                            detail: e.to_string(),
9628                            empty: false,
9629                        },
9630                    }
9631                }
9632            }
9633        };
9634
9635        self.state.status = match (mode, outcome.ok) {
9636            (MergeMode::None, _) => RunStatus::Ready,
9637            (_, true) if outcome.empty => RunStatus::Ready,
9638            (MergeMode::Pr, true) => RunStatus::Ready,
9639            (_, true) => RunStatus::Merged,
9640            (_, false) => RunStatus::Blocked,
9641        };
9642        self.state.event(
9643            "merge",
9644            format!(
9645                "{:?}: {}",
9646                mode,
9647                outcome.detail.lines().next().unwrap_or("")
9648            ),
9649        );
9650        self.state.merge = Some(outcome);
9651        self.state.save()?;
9652
9653        // The PR is open and the run would historically stop here, leaving the
9654        // operator to watch checks, feed review comments back to a fixer, and
9655        // merge. That was done by hand six times in one session before this
9656        // existed. Opt-in, because merging is the one irreversible thing magi
9657        // can do to a repository.
9658        if self.state.config.graph.land
9659            && mode == MergeMode::Pr
9660            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9661        {
9662            self.run_land().await?;
9663        }
9664        // `run_land` may have left `status` at `Landing` - still waiting on
9665        // CI or the owner's approval, not actually settled - so this has to
9666        // read whatever `status` ended up as here, not the `Merged` this
9667        // function set a few lines up.
9668        self.settle_questions();
9669        Ok(())
9670    }
9671
9672    /// Enter `land`.
9673    ///
9674    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9675    /// a resumed run's re-entry. `land::land` itself is what serialises the
9676    /// two git-mutating moments inside the loop — the rebase push and
9677    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9678    /// to hold a lock across the whole call, and doing so would serialise
9679    /// this run's CI wait against a *different* run's land-approval resume
9680    /// in the same repository, which is exactly the "must not wait on
9681    /// another task" property the daemon's slot-freeing exists to give.
9682    async fn run_land(&mut self) -> Result<()> {
9683        crate::pr_head::ensure_owner(&self.state)?;
9684        let url = self
9685            .state
9686            .pr
9687            .as_ref()
9688            .map(|pr| pr.url.clone())
9689            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9690            .unwrap_or_default();
9691        let url = url.lines().next().unwrap_or("").trim().to_owned();
9692        if !url.starts_with("http") {
9693            return Ok(());
9694        }
9695        // A land failure is not a lost run: the work is on a branch and the
9696        // pull request is open, which is exactly where a human takes over.
9697        loop {
9698            match land::land(&mut self.state, &url).await {
9699                Ok(_)
9700                    if self
9701                        .state
9702                        .merge_repairs
9703                        .last()
9704                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9705                {
9706                    if self.repair_merge_review().await? {
9707                        continue;
9708                    }
9709                }
9710                Ok(_) if self.state.resource_block.is_some() => {
9711                    return self.stop_for_resource();
9712                }
9713                Ok(pr) if self.state.parked => {
9714                    // `land` already saved the parked marker; nothing here
9715                    // overrides `status` back to a terminal value while an
9716                    // approval is still outstanding.
9717                    let _ = pr;
9718                }
9719                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9720                    self.state.status = RunStatus::Superseded;
9721                    self.state.save()?;
9722                }
9723                Ok(pr) => {
9724                    self.state.status = match pr.state {
9725                        land::PrLifecycle::Merged => RunStatus::Merged,
9726                        _ => RunStatus::Blocked,
9727                    };
9728                    // Downstream of a confirmed merge only - see
9729                    // `bump::should_release_bump`'s own doc for why this one
9730                    // check covers all three of `land`'s success paths.
9731                    // Best-effort: the run already landed, so a failure here
9732                    // (the decision call, `gh`, `cargo`) is recorded and never
9733                    // turns a landed run into a failed one.
9734                    if bump::should_release_bump(self.state.status)
9735                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9736                    {
9737                        // The event is the run's own record. Not-eligible cases
9738                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9739                        // `Err` is a bump that was tried and failed:
9740                        // `after_merge` itself raises the operator notice for
9741                        // that, whether or not a release PR exists yet.
9742                        self.state
9743                            .event("bump", format!("release bump skipped: {e:#}"));
9744                    }
9745                    // Independent of the bump, and best-effort in the same way:
9746                    // findings the merge left open become follow-up tasks.
9747                    if self.state.status == RunStatus::Merged {
9748                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9749                    }
9750                    self.state.save()?;
9751                }
9752                Err(e) => {
9753                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9754                        return Err(e);
9755                    }
9756                    self.state.status = RunStatus::Blocked;
9757                    self.state.event("land", format!("gave up: {e}"));
9758                    self.state.save()?;
9759                }
9760            }
9761            break;
9762        }
9763        Ok(())
9764    }
9765
9766    /// One repair cycle stays under the existing driver/task claim. An
9767    /// interrupted fixer is rolled back and held rather than run twice.
9768    async fn repair_merge_review(&mut self) -> Result<bool> {
9769        use crate::run::MergeRepairPhase as Phase;
9770        crate::pr_head::ensure_owner(&self.state)?;
9771        let i = self.state.merge_repairs.len() - 1;
9772        let request = self.state.merge_repairs[i].clone();
9773        if request.phase == Phase::Fixing {
9774            let restored = match self.state.winner().cloned() {
9775                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9776                None => Err(anyhow::anyhow!("repair has no winning branch")),
9777            };
9778            let why = match restored {
9779                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9780                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9781                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9782            };
9783            self.hold_merge_repair(i, why)?;
9784            return Ok(false);
9785        }
9786        // A completed repair gate is a publication checkpoint. On reentry,
9787        // reject external changes before the ordinary gate path can replace
9788        // that evidence by verifying a new head. Requested repairs still
9789        // start their own review generation below.
9790        if request.phase == Phase::Reviewing && self.state.gate_ran {
9791            let winner = self
9792                .state
9793                .winner()
9794                .context("repair has no winning branch")?;
9795            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9796            if self.state.gate_head.as_deref() != Some(head.as_str())
9797                || !git::is_clean(&winner.worktree).await?
9798            {
9799                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9800                return Ok(false);
9801            }
9802        }
9803        let failure = if i >= self.state.config.graph.land_rounds
9804            || self.state.config.graph.review_rounds == 0
9805        {
9806            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9807        } else {
9808            None
9809        };
9810        if let Some(why) = failure {
9811            self.hold_merge_repair(i, why)?;
9812            return Ok(false);
9813        }
9814        if request.phase == Phase::Requested {
9815            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9816                Ok(pr) => pr,
9817                Err(e) => {
9818                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9819                    return Ok(false);
9820                }
9821            };
9822            self.state.merge_repairs[i].phase = Phase::Fixing;
9823            self.state.land_approval = None;
9824            self.state.land_armed_head = None;
9825            self.state.save()?;
9826            // Serialize the exact records, including finding ids/details,
9827            // reconsideration reasons, final votes and verification output.
9828            let evidence = serde_json::to_string_pretty(&request)?;
9829            let reason = format!(
9830                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9831            );
9832            let budget = self.state.config.graph.land_rounds;
9833            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9834            match result {
9835                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9836                Ok(land::Fixed::Deferred) => {
9837                    // An unavailable fixer did not spend this repair request.
9838                    // Restore partial fallback work before retrying it, as for
9839                    // an interrupted repair, without closing the request.
9840                    let worktree = self
9841                        .state
9842                        .winner()
9843                        .context("repair has no winning branch")?
9844                        .worktree
9845                        .clone();
9846                    self.graft_restore(&worktree, &request.head).await?;
9847                    self.state.merge_repairs[i].phase = Phase::Requested;
9848                    self.stop_for_resource()?;
9849                    return Ok(false);
9850                }
9851                Ok(land::Fixed::Failed(why)) => {
9852                    self.hold_merge_repair(
9853                        i,
9854                        format!(
9855                            "repair failed: {why}; inspect the branch and run an explicit review"
9856                        ),
9857                    )?;
9858                    return Ok(false);
9859                }
9860                Err(e) => {
9861                    self.hold_merge_repair(
9862                        i,
9863                        format!(
9864                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9865                        ),
9866                    )?;
9867                    return Ok(false);
9868                }
9869            }
9870            // Even no change gets a real new reviewer wave. The previous
9871            // evidence remains in the request; a former clean/concluded round
9872            // cannot short-circuit the new review or gate.
9873            self.begin_merge_repair_review(i)?;
9874        }
9875        self.review_loop().await?;
9876        self.gate().await?;
9877        if self.resource_halt()? || self.state.parked {
9878            return Ok(false);
9879        }
9880        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9881            != Some(RunStatus::Gating)
9882            || !self.state.gate_allows_progress()
9883        {
9884            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9885            return Ok(false);
9886        }
9887        // Review and gate may have added commits after the initial repair.
9888        // Publish exactly their final head to the existing PR, with a lease.
9889        let winner = self
9890            .state
9891            .winner()
9892            .cloned()
9893            .context("repair has no winning branch")?;
9894        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9895        if self.state.gate_head.as_deref() != Some(after.as_str())
9896            || !git::is_clean(&winner.worktree).await?
9897        {
9898            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9899            return Ok(false);
9900        }
9901        let expected = self
9902            .state
9903            .pr_pushed_head
9904            .as_deref()
9905            .unwrap_or(&request.head)
9906            .to_owned();
9907        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9908        // Push the checked commit itself: a local ref moving after the check
9909        // must not change which commit the forge receives.
9910        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9911            return Ok(false);
9912        }
9913        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9914        let refspec = format!("{after}:refs/heads/{public}");
9915        let push = git::git_raw(
9916            &winner.worktree,
9917            &["push", &lease, &self.state.config.merge.remote, &refspec],
9918        )
9919        .await;
9920        match push {
9921            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9922            Ok(out) => {
9923                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9924                return Ok(false);
9925            }
9926            Err(e) => {
9927                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9928                return Ok(false);
9929            }
9930        }
9931        self.state.merge_repairs[i].phase = Phase::Complete;
9932        self.state.status = RunStatus::Landing;
9933        self.state.event(
9934            "repair",
9935            "new review and gate completed; fresh owner merge decision required",
9936        );
9937        self.state.save()?;
9938        Ok(true)
9939    }
9940
9941    fn merge_repair_artifacts(&self) -> PathBuf {
9942        let artifacts = agent::artifacts_dir(&self.state.dir());
9943        match self
9944            .state
9945            .merge_repairs
9946            .last()
9947            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9948        {
9949            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9950            None => artifacts,
9951        }
9952    }
9953
9954    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9955        if let Some(e2e) = self.state.head_e2e.take() {
9956            self.state
9957                .verification_history
9958                .push(crate::run::VerificationEvidence {
9959                    superseded_by: None,
9960                    archived_at: Some(Timestamp::now()),
9961                    fixes_spent: Some(self.state.gate_fixes.len()),
9962                    gate_head: None,
9963                    gate_ran: false,
9964                    gate: Vec::new(),
9965                    e2e: Some(e2e),
9966                });
9967        }
9968        self.state.reviews.clear();
9969        self.state.gate.clear();
9970        self.state.gate_ran = false;
9971        self.state.gate_head = None;
9972        self.state.gate_fixes.clear();
9973        self.state.gate_repair_e2e = None;
9974        self.state.pre_gate.clear();
9975        self.state.pre_gate_commit = None;
9976        self.state.contested_handoff = None;
9977        self.state.land_reruns = None;
9978        self.state.land_approval = None;
9979        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9980        self.state.status = RunStatus::Reviewing;
9981        self.state.save()
9982    }
9983
9984    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9985        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9986        self.state.merge_repairs[index].failure = Some(why.clone());
9987        self.state.status = RunStatus::Blocked;
9988        self.state.event("repair", why);
9989        self.state.save()
9990    }
9991
9992    // -------------------------------------------------------------- helpers
9993
9994    /// Fetch or create a seat, keeping its conversation across nodes.
9995    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9996        if let Some(existing) = self.state.seats.get(key)
9997            && existing.agent == agent
9998        {
9999            return existing.clone();
10000        }
10001        // A seat that changes agent mints its session id from the agent too,
10002        // like a handover: the old agent's uuid is already taken by the CLI.
10003        let fresh = if self.state.seats.contains_key(key) {
10004            handover_seat(key, agent, self.state.next_seat_seed())
10005        } else {
10006            SeatState::new(key, agent, self.state.seed)
10007        };
10008        self.state.seats.insert(key.to_owned(), fresh.clone());
10009        fresh
10010    }
10011
10012    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10013    /// seat to another roster agent, in which case that agent. Nodes that
10014    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10015    /// reconsideration) must keep talking to whoever answered it, not slip
10016    /// back to the agent that failed it.
10017    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10018        match self.state.seats.get(key) {
10019            Some(s) if s.agent != spec.id => {
10020                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10021            }
10022            _ => spec,
10023        }
10024    }
10025
10026    /// A candidate rendered for judging, with the leak policy applied.
10027    fn view(&self, c: &Candidate) -> CandidateView {
10028        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10029            .unwrap_or_default();
10030        let (patch, _) = blind::sanitize_patch(
10031            &format!("candidate {} patch", c.label),
10032            &raw,
10033            &self.state.config.blind,
10034        );
10035        CandidateView {
10036            label: c.label,
10037            branch: c.branch.clone(),
10038            summary: c.summary.clone(),
10039            stat: c.stat.clone(),
10040            patch,
10041        }
10042    }
10043
10044    /// The full candidate set as prompt text, for seats with no live session.
10045    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10046        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10047        prompt::judge(
10048            "(see above)",
10049            &views,
10050            self.roles.judges.len(),
10051            base_short,
10052            "en",
10053        )
10054    }
10055
10056    /// The final-vote prompt with everything a seat that has no session of its
10057    /// own needs: the candidates, the seat's own ranking and reasons, and the
10058    /// anonymised deliberation it took part in (only when there was one, so a
10059    /// handed-over seat never sees more than the seat it replaces did). The
10060    /// `Final vote` heading stays first.
10061    fn vote_prompt_full(
10062        &self,
10063        j: usize,
10064        labels: &[char],
10065        language: &str,
10066        candidates: &[Candidate],
10067        base_short: &str,
10068    ) -> String {
10069        let mut text = format!(
10070            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10071            prompt::final_vote(labels, language),
10072            self.state.instruction,
10073            self.candidate_block(candidates, base_short)
10074        );
10075        if let Some(own) = self
10076            .state
10077            .judgements
10078            .get(j)
10079            .filter(|r| !r.ranking.is_empty())
10080        {
10081            let reasons = own
10082                .reasons
10083                .iter()
10084                .map(|(k, v)| format!("- {k}: {v}"))
10085                .collect::<Vec<_>>()
10086                .join("\n");
10087            text.push_str(&format!(
10088                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10089                own.ranking.iter().collect::<String>(),
10090                if reasons.is_empty() {
10091                    ""
10092                } else {
10093                    ", because:\n"
10094                }
10095            ));
10096        }
10097        if !self.state.deliberation.is_empty() {
10098            text.push_str("\n# What was argued before this vote\n");
10099            for t in self.transcript(&[], j) {
10100                text.push_str(&format!(
10101                    "\n## {}{}\n\n{}\n",
10102                    t.who,
10103                    if t.is_self { " (you)" } else { "" },
10104                    t.body.trim()
10105                ));
10106            }
10107        }
10108        text
10109    }
10110
10111    /// Anonymised transcript for judge `self_idx`.
10112    ///
10113    /// The initial rankings are always the opening statements. Seeding them
10114    /// only when no turn had been taken yet meant every judge after the first
10115    /// argued against a single voice instead of against the actual split — the
10116    /// disagreement is the information, so it is always on the table.
10117    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10118        let mut turns = Vec::new();
10119        for j in &self.state.judgements {
10120            if j.ranking.is_empty() {
10121                continue;
10122            }
10123            let reasons = j
10124                .reasons
10125                .iter()
10126                .map(|(k, v)| format!("- {k}: {v}"))
10127                .collect::<Vec<_>>()
10128                .join("\n");
10129            turns.push(Turn {
10130                who: format!("Judge {} (opening ranking)", j.judge),
10131                is_self: j.judge == self_idx + 1,
10132                body: format!(
10133                    "Ranked {}{}{reasons}",
10134                    j.ranking.iter().collect::<String>(),
10135                    if reasons.is_empty() {
10136                        ""
10137                    } else {
10138                        ", because:\n"
10139                    }
10140                ),
10141            });
10142        }
10143        for t in self
10144            .state
10145            .deliberation
10146            .iter()
10147            .flat_map(|r| r.turns.iter())
10148            .chain(current)
10149        {
10150            turns.push(Turn {
10151                who: format!("Judge {}", t.judge),
10152                is_self: t.judge == self_idx + 1,
10153                body: t.body.clone(),
10154            });
10155        }
10156        turns
10157    }
10158}
10159
10160/// Does this seat still hold the context a follow-up prompt would rely on?
10161fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10162    agent::has_session(spec.kind, seat, sessions)
10163}
10164
10165/// The next entry in `roster` after `start`, never wrapping back to the
10166/// front, whose id is not in `tried` yet.
10167///
10168/// Starts one past `start` rather than at the front of `roster`: `start` is
10169/// the seat's own original position, and a seat whose candidate slot already
10170/// sits on the roster's second entry must fall through to the third next, not
10171/// restart at the first — which is very likely a different candidate's own
10172/// agent already. Never wraps back past `start`, for the same reason: an
10173/// entry earlier in the roster than the seat's own position is almost
10174/// certainly some *other* candidate slot's own agent, and once the tail of
10175/// the roster is exhausted there are no more untried agents for *this* seat
10176/// to fall through to — the caller's fallback chain ends there, exactly as
10177/// "no further untried agents remain in the list for that seat" asks for.
10178///
10179/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10180/// the same id twice (an operator's `roles.implementers` typo, or a
10181/// `[[agents]]` list reused across roles) must not let
10182/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10183/// over `roster` either finds an untried id or runs out, so this always
10184/// terminates regardless of duplicates.
10185fn next_untried_in_roster<'a>(
10186    roster: &'a [AgentSpec],
10187    start: usize,
10188    tried: &BTreeSet<String>,
10189) -> Option<&'a AgentSpec> {
10190    roster
10191        .get(start + 1..)?
10192        .iter()
10193        .find(|s| !tried.contains(&s.id))
10194}
10195
10196/// The successor for a seat, shared by all four seat kinds (implement, judge,
10197/// review, advise). `others` holds the ids that *currently* occupy the other
10198/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10199/// sees them), so roster entries beyond the seat count act as spares: a failed
10200/// seat goes to an agent no other seat holds whenever the roster permits.
10201///
10202/// `carried` is `Some` only for the review loop's carried failure history.
10203/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10204/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10205/// whole roster: untried and not another seat's occupant; (3) the plain walk
10206/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10207/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10208/// a better outcome than an empty seat, and it keeps the answer to "is there
10209/// a successor at all" exactly what it was before occupants were considered,
10210/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10211/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10212/// carried failure is a cheaper bet than doubling an agent on the panel, and
10213/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10214/// the same round are handled one at a time, so a seat later in the batch
10215/// sees an earlier one's new occupant but not yet its own freed agent.
10216fn pick_successor<'a>(
10217    roster: &'a [AgentSpec],
10218    start: usize,
10219    tried: &BTreeSet<String>,
10220    carried: Option<&BTreeSet<String>>,
10221    others: &BTreeSet<String>,
10222) -> Option<&'a AgentSpec> {
10223    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10224    roster
10225        .get(start + 1..)
10226        .and_then(|tail| {
10227            tail.iter()
10228                .filter(free)
10229                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10230        })
10231        .or_else(|| {
10232            carried
10233                .is_some()
10234                .then(|| roster.iter().find(free))
10235                .flatten()
10236        })
10237        .or_else(|| match carried {
10238            Some(c) => next_for_seat(roster, start, tried, c),
10239            None => next_untried_in_roster(roster, start, tried),
10240        })
10241}
10242
10243/// The next agent for a seat that carries its failure history across rounds
10244/// (the review loop). `round_tried` is this round's own bound and starts
10245/// empty every round; `carried_failed` only decides priority.
10246///
10247/// First: an id walking forward from `start`, never wrapping, that is neither
10248/// tried this round nor failed in an earlier one. Only when that is exhausted
10249/// does it rescue: the first roster id (in roster order, so this one *does*
10250/// look before `start`) not yet tried this round, which is by then a carried
10251/// failure. Each id is rescued at most once per round, so it cannot loop.
10252fn next_for_seat<'a>(
10253    roster: &'a [AgentSpec],
10254    start: usize,
10255    round_tried: &BTreeSet<String>,
10256    carried_failed: &BTreeSet<String>,
10257) -> Option<&'a AgentSpec> {
10258    roster
10259        .get(start + 1..)?
10260        .iter()
10261        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10262        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10263}
10264
10265/// Where a reviewer seat starts a round: the agent that last answered it when
10266/// it is still on the roster and not marked failed, else the spec's own agent
10267/// unless it failed, else the next roster agent that has not failed, else the
10268/// spec's own agent again (the whole roster failed). Ids no longer on the
10269/// roster are ignored. An empty roster has no handover, so the spec stands.
10270fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10271    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10272        return spec;
10273    };
10274    let ok = |id: &str| !h.failed.contains(id);
10275    if let Some(last) = h.last_ok.as_deref()
10276        && ok(last)
10277        && let Some(s) = roster.iter().find(|s| s.id == last)
10278    {
10279        return s.clone();
10280    }
10281    if ok(&spec.id) {
10282        return spec;
10283    }
10284    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10285    roster
10286        .get(from + 1..)
10287        .into_iter()
10288        .flatten()
10289        .chain(roster.iter())
10290        .find(|s| ok(&s.id))
10291        .cloned()
10292        .unwrap_or(spec)
10293}
10294
10295/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10296/// seed so a CLI that mints its session id up front (`--session-id`) never
10297/// reuses the uuid the previous agent already opened under the same seat key.
10298pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10299    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10300}
10301
10302/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10303/// so the classifier below and the code that builds the message cannot drift.
10304const TIMED_OUT: &str = "timed out";
10305
10306impl FailClass {
10307    /// `None` for an answer; otherwise how the turn failed.
10308    fn of(out: &AgentOutcome) -> Option<Self> {
10309        match out {
10310            AgentOutcome::Ok(_) => None,
10311            AgentOutcome::Quota(_) => Some(Self::Quota),
10312            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10313            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10314            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10315            // Not a failure of the agent. Callers check for it first; `None`
10316            // here means "no class to hand over on".
10317            AgentOutcome::ResourceBlocked(_) => None,
10318        }
10319    }
10320
10321    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10322    fn stem_word(&self) -> &'static str {
10323        match self {
10324            Self::Quota => "quota",
10325            _ => "handover",
10326        }
10327    }
10328}
10329
10330/// The message's first line with its variable parts removed — digit runs and
10331/// path-like tokens — so "exit code 2" and "exit code 7" read
10332/// as one kind of failure.
10333fn failure_signature(msg: &str) -> String {
10334    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10335    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10336    // the historical machine key while the process diagnosis uses readable prose.
10337    if let Some(code) = line
10338        .strip_prefix("process exited with no usable output (exit code ")
10339        .and_then(|s| s.strip_suffix(')'))
10340        .and_then(|s| s.parse::<i32>().ok())
10341    {
10342        line = format!("exited with some({code}) and no usable output");
10343    } else if matches!(
10344        line.as_str(),
10345        "process exited with no usable output (no exit code reported)"
10346            | "process exited with no usable output (exit code unavailable)"
10347    ) {
10348        line = "exited with none and no usable output".to_owned();
10349    }
10350    let mut out = Vec::new();
10351    for word in line.split_whitespace() {
10352        if word.contains('/') || word.contains('\\') {
10353            out.push("<path>".to_owned());
10354            continue;
10355        }
10356        let mut w = String::new();
10357        let mut in_digits = false;
10358        for c in word.chars() {
10359            if c.is_ascii_digit() {
10360                if !in_digits {
10361                    w.push('#');
10362                }
10363                in_digits = true;
10364            } else {
10365                in_digits = false;
10366                w.push(c);
10367            }
10368        }
10369        out.push(w);
10370    }
10371    out.join(" ").chars().take(120).collect()
10372}
10373
10374/// Whether a seat that just failed with `cur` may go to the next roster agent.
10375/// A quota or a timeout always may. Any other failure may not when the agent
10376/// before it failed the same way: an error the prompt causes would otherwise
10377/// walk the whole roster. `prev` is the class of the immediately preceding
10378/// agent's failure, so a quota or timeout in between breaks the run of
10379/// identical failures by itself.
10380fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10381    match cur {
10382        FailClass::Quota | FailClass::Timeout => true,
10383        FailClass::Other(_) => prev != Some(cur),
10384    }
10385}
10386
10387/// A short human reason for a failed outcome, for the handover record.
10388fn fail_reason(out: &AgentOutcome) -> String {
10389    match out {
10390        AgentOutcome::Ok(_) => String::new(),
10391        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10392        AgentOutcome::Dropped(o) => format!(
10393            "the CLI dropped the stream ({})",
10394            o.dropped
10395                .as_ref()
10396                .map(|d| d.why.as_str())
10397                .unwrap_or("it ended without delivering its answer")
10398        ),
10399        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10400            e.lines().next().unwrap_or("").chars().take(160).collect()
10401        }
10402    }
10403}
10404
10405/// Note one handover in the run: the structured record and, in the timeline,
10406/// the sentence a person reads. A quota keeps the wording it always had.
10407pub(crate) fn record_handover(
10408    state: &mut RunState,
10409    node: &str,
10410    seat: &str,
10411    from: &str,
10412    to: &str,
10413    class: &FailClass,
10414    reason: &str,
10415) {
10416    let message = if *class == FailClass::Quota {
10417        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10418    } else {
10419        format!("{seat}: handed over {from} -> {to} ({reason})")
10420    };
10421    state.event(node, message);
10422    state.handovers.push(Handover {
10423        at: Timestamp::now(),
10424        node: node.to_owned(),
10425        seat: seat.to_owned(),
10426        from: from.to_owned(),
10427        to: to.to_owned(),
10428        reason: reason.to_owned(),
10429    });
10430}
10431
10432/// Did this reply report running a command whose own CLI never confirmed an
10433/// exit status?
10434///
10435/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10436/// command *finished* (see that type's own doc), so this can only be `true`
10437/// for a command whose completion event carried no readable exit code — not
10438/// for one that simply is not mentioned at all. That is the one signal this
10439/// crate can read, from the same record `state.jobs` renders, about a reply
10440/// standing next to work its own CLI cannot vouch for finishing; it is
10441/// deliberately not a check on the exit code's *value* (a fixer legitimately
10442/// runs a command that fails mid-iteration before it succeeds) and not a
10443/// guess at a command still running in the background (which emits no event
10444/// at all, and so leaves no evidence here to find).
10445fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10446    commands.iter().any(|c| c.exit_code.is_none())
10447}
10448
10449/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10450/// trusted as a verified no-op — the adoption guard's own text-level half.
10451///
10452/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10453/// timed out): a marker only earns the benefit of the doubt from a turn the
10454/// CLI itself vouches for finishing properly, the same house style
10455/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10456/// *fix* report to for `commands`. A candidate that timed out, exited
10457/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10458/// is, whatever prose it wrote — this returns `None` before it ever looks at
10459/// `text`. The remaining guards (the tree really is empty, the evidence is
10460/// non-empty) are the caller's: this only reads what the reply *claimed*.
10461fn verified_noop_claim(
10462    usable: bool,
10463    commands: &[agent::CommandEvidence],
10464    text: &str,
10465) -> Option<String> {
10466    (usable && !has_unconfirmed_command(commands))
10467        .then(|| verdict::verified_noop(text))
10468        .flatten()
10469}
10470
10471fn short(commit: &str) -> String {
10472    commit.chars().take(7).collect()
10473}
10474
10475fn make_executable(path: &Path) -> Result<()> {
10476    #[cfg(unix)]
10477    {
10478        use std::os::unix::fs::PermissionsExt as _;
10479        let mut perms = std::fs::metadata(path)?.permissions();
10480        perms.set_mode(0o755);
10481        std::fs::set_permissions(path, perms)?;
10482    }
10483    #[cfg(not(unix))]
10484    {
10485        let _ = path;
10486    }
10487    Ok(())
10488}
10489
10490/// What every seat in one batch shares: where the answers are attributed, the
10491/// prompt overlay they inherit, and the build cache they are told to use.
10492///
10493/// A struct rather than four more parameters: `wave` also needs the run's
10494/// state (to record who is answering right now) and the attempt number, and
10495/// eight positional arguments is both unreadable and a clippy error.
10496struct WaveCtx<'a> {
10497    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10498    /// paid for it.
10499    run: &'a str,
10500    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10501    node: &'a str,
10502    prompts: &'a Prompts,
10503    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10504    cache: Option<&'a Path>,
10505    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10506    /// `JobRecord::round`. `None` for every other node.
10507    round: Option<usize>,
10508    /// Carry each seat's failed-agent history across waves (the review loop
10509    /// only): start-of-round priority and handover choice read
10510    /// [`RunState::seat_history`], and every answer or failure writes it.
10511    carry_seats: bool,
10512    /// The run's park / interrupt requests, so a wait for a shared resource
10513    /// can be abandoned the moment the operator asks to park.
10514    pauses: Option<(&'a Pause, &'a Pause)>,
10515}
10516
10517/// Shared scheduling policy for operations and spawned seat invocations.
10518fn repository_wait(
10519    state: &RunState,
10520    pauses: Option<(&Pause, &Pause)>,
10521) -> crate::repo_lock::WaitPolicy {
10522    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10523    let inherited = crate::repo_lock::WaitPolicy::current();
10524    crate::repo_lock::WaitPolicy::new(
10525        Duration::from_secs(state.config.graph.timeout_resource_wait),
10526        move || {
10527            inherited.cancelled()
10528                || pauses
10529                    .as_ref()
10530                    .is_some_and(|(a, b)| a.parked() || b.parked())
10531        },
10532    )
10533}
10534
10535fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10536    if error.is::<crate::repo_lock::ResourceBlocked>() {
10537        state.resource_block = Some(crate::run::ResourceBlock {
10538            cause: crate::run::ResourceBlockCause::Other,
10539            node: "repository".into(),
10540            kind: if error.to_string().contains("disk pressure") {
10541                "disk pressure"
10542            } else if error.to_string().contains("disk admission") {
10543                "disk admission"
10544            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10545                "incremental cache admission"
10546            } else {
10547                "repository coordination"
10548            }
10549            .into(),
10550            detail: error.to_string(),
10551            at: Timestamp::now(),
10552        });
10553        state.event("resource", error.to_string());
10554    }
10555}
10556
10557async fn coordinate_operation(
10558    state: &mut RunState,
10559    cwd: &Path,
10560    wait: &crate::repo_lock::WaitPolicy,
10561) -> Result<Option<crate::repo_lock::SeatLease>> {
10562    let result =
10563        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10564            .await;
10565    if let Err(e) = &result {
10566        record_repository_block(state, e);
10567    }
10568    result
10569}
10570
10571/// Run one job, honouring the parallelism budget.
10572async fn run_one(
10573    job: SeatJob,
10574    sem: Arc<Semaphore>,
10575    ctx: &WaveCtx<'_>,
10576    state: &mut RunState,
10577    attempt: usize,
10578) -> (SeatState, AgentOutcome) {
10579    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10580        .await
10581        .pop()
10582        .expect("one job in, one result out");
10583    (seat, out)
10584}
10585
10586/// Run every job concurrently, capped by the semaphore, preserving order.
10587///
10588/// Every seat in the batch is recorded into [`RunState::active`] before the
10589/// wave starts and cleared as each answer lands, so the run's own record says
10590/// who is still being waited on rather than only who finished.
10591async fn wave(
10592    jobs: Vec<SeatJob>,
10593    sem: Arc<Semaphore>,
10594    ctx: &WaveCtx<'_>,
10595    state: &mut RunState,
10596    attempt: usize,
10597) -> Vec<(usize, SeatState, AgentOutcome)> {
10598    let WaveCtx {
10599        run,
10600        node,
10601        prompts,
10602        cache,
10603        round,
10604        carry_seats: _,
10605        pauses,
10606    } = *ctx;
10607    for job in &jobs {
10608        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10609    }
10610    if let Err(e) = state.save() {
10611        // A failed persist of "who is answering right now" must not abort the
10612        // wave: the seats are already being asked, and the alternative is
10613        // losing the answers to save a status line nobody may even be
10614        // watching.
10615        tracing::warn!("could not persist in-progress seats: {e:#}");
10616    }
10617    // Hold the shared build cache's lease for the whole batch, not per job:
10618    // several candidates (an implement wave) or a fixer legitimately share
10619    // one cache concurrently within this run, and that stays untouched — a
10620    // single lease taken once for the whole wave and released once it is
10621    // done is what stops a *different* borrower (another run's own wave, its
10622    // e2e/gate, a human's `magi review`) from interleaving a build into the
10623    // same directory while this one is in flight. Best-effort, not
10624    // all-or-nothing: a wave that cannot get the lease within its own
10625    // longest job's budget still runs — an hour of paid implementer calls is
10626    // not thrown away over cache contention — but every write-allowed seat
10627    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10628    // below), the same fallback a read-only seat always gets, rather than
10629    // building into a directory this run was never granted. The identity
10630    // record is still invalidated below either way, so the next tracked
10631    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10632    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10633    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10634    // never borrowed from a job's agent budget (and never charged to it either,
10635    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10636    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10637    let cache_guard = if let Some(cache_dir) = cache {
10638        if jobs_had_a_writer {
10639            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10640            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10641            match acquire_cache_lease(
10642                state,
10643                cache_dir,
10644                &owner,
10645                resource_limit,
10646                node,
10647                node,
10648                &cancel,
10649            )
10650            .await
10651            {
10652                Ok(Some(g)) => Some(g),
10653                // Parking: leave without spawning anything. The run parks at
10654                // the next boundary and the seats are asked afresh on resume.
10655                Ok(None) => {
10656                    if let Some(block) = &mut state.resource_block {
10657                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10658                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10659                        };
10660                    }
10661                    return block_wave(
10662                        jobs,
10663                        state,
10664                        node,
10665                        attempt,
10666                        "parked while waiting for the shared build cache",
10667                    );
10668                }
10669                // An unreadable lease is not contention: keep the old
10670                // fallback (no shared cache for this wave's seats).
10671                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10672                Err(e) => {
10673                    // Writing seats may not build into a cache this run does
10674                    // not hold, and an agent that cannot be given one is not
10675                    // asked at all: the run stops resumably instead.
10676                    let why = format!("{e:#}");
10677                    state.event(
10678                        "resource",
10679                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10680                    );
10681                    return block_wave(jobs, state, node, attempt, &why);
10682                }
10683            }
10684        } else {
10685            None
10686        }
10687    } else {
10688        None
10689    };
10690    // Every job keeps its whole budget: the wait above was not the agent's.
10691    // Restart the seats' clocks so "started" means "being spawned now".
10692    state.seats_spawning(node, attempt);
10693    if let Err(e) = state.save() {
10694        tracing::warn!("could not persist the start of a wave: {e:#}");
10695    }
10696    // Queueing behind this wave's own seats for a parallelism slot is normal
10697    // (each of them is bounded by its own agent timeout); only a wait beyond
10698    // all of them plus the resource limit means the slots never came back.
10699    let slot_limit = resource_limit
10700        + jobs
10701            .iter()
10702            .map(|j| j.timeout)
10703            .max()
10704            .unwrap_or_default()
10705            .saturating_mul(jobs.len() as u32);
10706    let mut set = tokio::task::JoinSet::new();
10707    // Seats announce themselves here the moment they hold a parallelism
10708    // permit, i.e. just before their CLI is spawned.
10709    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10710    let overlay = prompts.overlay(node);
10711    let repository_wait = repository_wait(state, pauses);
10712    for (i, mut job) in jobs.into_iter().enumerate() {
10713        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10714        if cache.is_some() {
10715            job.prompt.push('\n');
10716            job.prompt
10717                .push_str(&prompt::build_cache_note(node, job.allow_write));
10718        }
10719        let sem = Arc::clone(&sem);
10720        let run = run.to_owned();
10721        let node = node.to_owned();
10722        // Only implementers were told about the task's attachments, so only
10723        // their seats get the directory widened for reading.
10724        let attachments = if node == "implement" {
10725            state.attachments.clone()
10726        } else {
10727            Vec::new()
10728        };
10729        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10730        // `prompt::build_cache_note`'s doc for why setting it anyway is
10731        // exactly how a sandboxed reviewer's write refusal got reported as a
10732        // defect in the patch, not a property of its own seat. And a
10733        // write-allowed one is handed it only when the lease above was
10734        // actually acquired: a wave that could not get it (`cache_guard` is
10735        // `None`, see its own comment) must not send seats to build into a
10736        // directory this run does not hold - that is the exact concurrent,
10737        // unmanaged-write race this module exists to prevent, not something
10738        // "proceeding anyway" is allowed to reintroduce.
10739        let cache = cache
10740            .filter(|_| job.allow_write && cache_guard.is_some())
10741            .map(Path::to_path_buf);
10742        let spawned_tx = spawned_tx.clone();
10743        let repository_wait = repository_wait.clone();
10744        let coordination = state.config.coordination.clone();
10745        let fetched_base = state.fetched_base.clone();
10746        set.spawn(async move {
10747            // A wait for a parallelism slot is scheduling, not the agent: it
10748            // is bounded by its own limit and ends as a resource block, never
10749            // as a timeout of the agent that was not started.
10750            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10751                Ok(p) => p,
10752                Err(_) => {
10753                    return (
10754                        i,
10755                        job.seat,
10756                        AgentOutcome::ResourceBlocked(format!(
10757                            "no parallel slot became free within {}s",
10758                            slot_limit.as_secs()
10759                        )),
10760                        false,
10761                    );
10762                }
10763            };
10764            spawned_tx.send(job.seat.key.clone()).ok();
10765            let mut seat = job.seat;
10766            let out = repository_wait
10767                .scope(agent::invoke(
10768                    &job.spec,
10769                    &mut seat,
10770                    &Invocation {
10771                        coordination: coordination.as_ref(),
10772                        fetched_base: fetched_base.as_ref(),
10773                        cwd: &job.cwd,
10774                        prompt: &job.prompt,
10775                        timeout: job.timeout,
10776                        allow_write: job.allow_write,
10777                        sessions: job.sessions,
10778                        artifacts: &job.artifacts,
10779                        stem: &job.stem,
10780                        run: &run,
10781                        node: &node,
10782                        cache_dir: cache.as_deref(),
10783                        attachments: &attachments,
10784                        writable: &[],
10785                    },
10786                ))
10787                .await;
10788            let interrupted = out
10789                .as_ref()
10790                .err()
10791                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10792            let out = match out {
10793                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10794                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10795                // Billed work the CLI failed to hand over is not an ordinary
10796                // failure, but its text is the CLI's raw error JSON, not an
10797                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10798                // read it as one by forgetting to check. `usable()` is always
10799                // false here (dropped implies an empty response), so this has
10800                // to be checked before the catch-all `Failed` below or the
10801                // one shape this exists for is lost with the rest.
10802                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10803                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10804                Ok(o) => AgentOutcome::Failed(format!(
10805                    "process exited with no usable output ({})",
10806                    crate::notices::exit_status(o.exit_code)
10807                )),
10808                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10809                    AgentOutcome::ResourceBlocked(e.to_string())
10810                }
10811                Err(e) => AgentOutcome::Failed(e.to_string()),
10812            };
10813            (i, seat, out, interrupted)
10814        });
10815    }
10816    drop(spawned_tx);
10817    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10818    loop {
10819        let joined = tokio::select! {
10820            Some(key) = spawned_rx.recv() => {
10821                // The clock of this seat starts now, not when it queued.
10822                state.seat_spawned(&key);
10823                if let Err(e) = state.save() {
10824                    tracing::warn!("could not persist a seat's start: {e:#}");
10825                }
10826                continue;
10827            }
10828            joined = set.join_next() => joined,
10829        };
10830        let Some(joined) = joined else {
10831            break;
10832        };
10833        let (i, seat, out, interrupted) = match joined {
10834            Ok(v) => v,
10835            // No seat to clear: a panicked task never reported which one it
10836            // was. The defensive sweep below this loop is what stops that
10837            // seat's `active` entry from surviving forever.
10838            Err(e) => {
10839                tracing::error!("agent task panicked: {e}");
10840                continue;
10841            }
10842        };
10843        state.seat_finished(&seat.key);
10844        if interrupted
10845            && node == "implement"
10846            && let Some(label) = state
10847                .candidates
10848                .iter()
10849                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10850                .map(|candidate| candidate.label)
10851        {
10852            state.candidate_preservation.insert(
10853                label,
10854                "interrupted implementation awaiting persistence".into(),
10855            );
10856            state.interrupted_implementations.insert(label);
10857        }
10858        if let AgentOutcome::ResourceBlocked(why) = &out
10859            && state.resource_block.is_none()
10860        {
10861            state.resource_block = Some(crate::run::ResourceBlock {
10862                cause: crate::run::ResourceBlockCause::Other,
10863                node: node.to_owned(),
10864                kind: if why.contains("disk pressure") {
10865                    "disk pressure"
10866                } else if why.contains("disk admission") {
10867                    "disk admission"
10868                } else if why.contains(crate::incremental::BLOCKED) {
10869                    "incremental cache admission"
10870                } else if why.contains("repository coordination") {
10871                    "repository coordination"
10872                } else {
10873                    "parallel slot"
10874                }
10875                .to_owned(),
10876                detail: why.clone(),
10877                at: Timestamp::now(),
10878            });
10879            let activity = if interrupted {
10880                "agent execution interrupted"
10881            } else {
10882                "no agent was started"
10883            };
10884            state.event("resource", format!("{node}: {activity} - {why}"));
10885        }
10886        record_jobs(state, node, round, &seat.key, &out);
10887        if let Err(e) = state.save() {
10888            tracing::warn!("could not persist a seat's completion: {e:#}");
10889        }
10890        if collected.len() <= i {
10891            collected.resize_with(i + 1, || None);
10892        }
10893        collected[i] = Some((i, seat, out));
10894    }
10895    // Belt-and-braces for the panic branch above: every seat this exact batch
10896    // started shares this `(node, attempt)` pair, and every seat that finished
10897    // normally already cleared itself, so anything left tagged with it here
10898    // can only be a panicked task's leftover. Cleared unconditionally rather
10899    // than left to read as still answering forever.
10900    if state
10901        .active
10902        .values()
10903        .any(|a| a.node == node && a.attempt == attempt)
10904    {
10905        state
10906            .active
10907            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10908        if let Err(e) = state.save() {
10909            tracing::warn!("could not persist the end of a wave: {e:#}");
10910        }
10911    }
10912    // Whether or not the lease above was actually held, several worktrees
10913    // may just have built into the cache with nothing here able to name one
10914    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10915    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10916    // clean it might not have strictly needed; trusting a stale match would
10917    // cost it a wrong answer.
10918    if let Some(cache_dir) = cache
10919        && jobs_had_a_writer
10920    {
10921        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10922    }
10923    if let Some(guard) = cache_guard {
10924        guard.release();
10925    }
10926    collected.into_iter().flatten().collect()
10927}
10928
10929/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10930/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10931/// this is data collection, not the fix-specific completion contract in
10932/// [`Runner::continue_fix_report`], and applies regardless of which node
10933/// asked.
10934///
10935/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10936/// evidence from; `Failed` does not, and correctly contributes nothing — a
10937/// timeout or crash is not itself evidence about a command the seat may have
10938/// started.
10939fn record_jobs(
10940    state: &mut RunState,
10941    node: &str,
10942    round: Option<usize>,
10943    seat: &str,
10944    out: &AgentOutcome,
10945) {
10946    let commands: &[agent::CommandEvidence] = match out {
10947        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10948        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10949    };
10950    let checked_at = Timestamp::now();
10951    for c in commands {
10952        state.jobs.push(JobRecord {
10953            node: node.to_owned(),
10954            round,
10955            seat: seat.to_owned(),
10956            id: c.id.clone(),
10957            description: c.description.clone(),
10958            checked_at,
10959            status: match c.exit_code {
10960                Some(0) => JobStatus::Completed,
10961                Some(_) => JobStatus::Failed,
10962                None => JobStatus::Unknown,
10963            },
10964            exit_code: c.exit_code,
10965            result_summary: c.result_summary.clone(),
10966            source: c.source.clone(),
10967        });
10968    }
10969}
10970
10971/// Is a review round clean, given how many reviewer seats answered against
10972/// how many the round expected?
10973///
10974/// A seat that never answered (timeout, crash, unparsable output) is not a
10975/// seat that read the patch and found nothing — treating it as such is
10976/// exactly the bug this function exists to close. Under the default `block`
10977/// policy a missing seat can never be clean; `warn` still requires the seats
10978/// that *did* answer to have found nothing blocking and verification to be
10979/// green.
10980///
10981/// `quota_missing` narrows that `block` default for exactly one cause of
10982/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10983/// a session limit lifts by the very next round buys nothing — the seat is
10984/// asked again with the same quota — so once every missing seat is accounted
10985/// for by a quota loss (and at least one seat *did* answer, so a decision has
10986/// something to rest on) the round is decided on the panel that could answer,
10987/// same as `warn` would. A panel that lost every seat to quota is not
10988/// decided here: `answered == 0` falls through to the existing `block`
10989/// fallback so a fully collapsed panel still waits rather than landing on no
10990/// review at all.
10991fn round_is_clean(
10992    blocking: usize,
10993    e2e_ok: bool,
10994    answered: usize,
10995    expected: usize,
10996    quota_missing: usize,
10997    policy: IncompleteReviewPolicy,
10998) -> bool {
10999    if blocking != 0 || !e2e_ok {
11000        return false;
11001    }
11002    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11003        return true;
11004    }
11005    answered > 0 && expected - answered <= quota_missing
11006}
11007
11008/// The review loop's own conclusion, derived entirely from its persisted
11009/// round records and the round budget that produced them — never from
11010/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11011/// recomputes the identical answer regardless of what an earlier node in the
11012/// same walk, or a previous walk, did to `status`.
11013///
11014/// `None` while more rounds remain to try, including when review never ran
11015/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11016/// gone clean, or the budget is spent, or the tree has stopped moving (see
11017/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11018///
11019/// - An incomplete panel that raised nothing is missing input, not a
11020///   verified tree — never a hand-off candidate, whatever verification said
11021///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11022/// - Otherwise, green e2e on the last round hands off (see
11023///   [`Runner::stop_reviewing`]); red e2e blocks.
11024///
11025/// A last round whose own verification is still `ResourceBlocked` — magi
11026/// itself never got a command to run, not evidence the patch is broken —
11027/// is neither: this returns `None` for it too, the same as "more rounds
11028/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11029/// handling of that shape) instead of this cheap recomputation guessing a
11030/// verdict a real attempt never produced.
11031pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11032    if max_rounds == 0
11033        || reviews.last().is_some_and(|r| {
11034            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11035        })
11036    {
11037        return Some(RunStatus::Gating);
11038    }
11039    let last = reviews.last()?;
11040    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11041    if reviews.len() < max_rounds && !stagnant {
11042        return None;
11043    }
11044    if last.incomplete() && last.blocking == 0 {
11045        return Some(RunStatus::Blocked);
11046    }
11047    if last.e2e_status() == E2eStatus::ResourceBlocked
11048        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11049    {
11050        return None;
11051    }
11052    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11053        RunStatus::Gating
11054    } else {
11055        RunStatus::Blocked
11056    })
11057}
11058
11059/// How long a re-ask may take, given the budget the first attempt had.
11060///
11061/// A `nudged` retry is a request to restate an answer the seat has already
11062/// worked out: it carries no new work, so it does not deserve the original
11063/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11064/// 133 seconds while a third sat for over ten minutes on a resumed session
11065/// holding 410 KB of prior output - and because the retry had inherited the
11066/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11067/// judging round whose other seats were long finished.
11068///
11069/// A quarter of the budget, with a floor so that a deliberately short timeout
11070/// does not collapse to nothing. A retry that re-sends the whole prompt
11071/// (because the seat kept no context) is the original job again, and keeps the
11072/// original budget.
11073pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11074    if nudged {
11075        (full / 4).max(Duration::from_secs(120)).min(full)
11076    } else {
11077        full
11078    }
11079}
11080
11081/// Run a wave and parse each reply, re-asking the seats whose reply was
11082/// unusable.
11083///
11084/// The re-ask is a nudge rather than the whole prompt again when the seat still
11085/// holds its conversation, which is the difference between a cheap retry and
11086/// paying for the entire candidate set twice.
11087///
11088/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11089/// successor in `roster` is handed to it instead of being re-asked: the
11090/// handover is the retry. A seat with no successor left (a single-agent
11091/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11092/// the asks to one seat in one node number at most
11093/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11094/// is asked once (a dropped stream is nudged first), and only the last agent
11095/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11096/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11097/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11098/// other failure stops the chain when the previous agent failed the same way
11099/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11100/// [`has_context`] is false and the job's own full prompt and full budget are
11101/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11102/// intermediate ones are not losses) and is returned as a failure like any
11103/// other absent seat — the caller decides whether the panel still has a
11104/// quorum. An empty `roster` disables handover: failures are nudged as they
11105/// always were, and a quota is simply lost. A reply that fails to parse or
11106/// validate is the prompt's doing and is only ever nudged, never handed over.
11107///
11108/// The returned [`SeatState`] names the agent that answered (or tried last).
11109#[allow(clippy::too_many_arguments)]
11110async fn ask_json_wave<T>(
11111    jobs: Vec<SeatJob>,
11112    sem: Arc<Semaphore>,
11113    retries: usize,
11114    roster: &[AgentSpec],
11115    ctx: &WaveCtx<'_>,
11116    losses: &mut Vec<QuotaLoss>,
11117    state: &mut RunState,
11118    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11119) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11120where
11121    T: serde::de::DeserializeOwned + Send + 'static,
11122{
11123    ask_wave_with(
11124        jobs,
11125        sem,
11126        retries,
11127        roster,
11128        ctx,
11129        losses,
11130        state,
11131        &|text: &str| {
11132            let v = verdict::extract_json::<T>(text)?;
11133            validate(&v)?;
11134            Ok(v)
11135        },
11136    )
11137    .await
11138}
11139
11140/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11141/// a node whose answer is prose (deliberation) shares the same handover,
11142/// failure classification, quota bookkeeping and bounds instead of a copy.
11143///
11144/// A seat handed to another roster agent is sent the job's `handover` prompt
11145/// (when it has one) rather than `prompt`: the new agent has no session, so a
11146/// resume-style prompt would be incomplete. That holds for the handover ask
11147/// and for every nudge to that agent whose `has_context` is false.
11148#[allow(clippy::too_many_arguments)]
11149async fn ask_wave_with<T>(
11150    jobs: Vec<SeatJob>,
11151    sem: Arc<Semaphore>,
11152    retries: usize,
11153    roster: &[AgentSpec],
11154    ctx: &WaveCtx<'_>,
11155    losses: &mut Vec<QuotaLoss>,
11156    state: &mut RunState,
11157    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11158) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11159where
11160    T: Send + 'static,
11161{
11162    let n = jobs.len();
11163    let originals: Vec<SeatJob> = jobs;
11164    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11165    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11166    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11167    // answer, N once it has gone through N nudges. Read back once this
11168    // returns, so a caller building a history record (`ReviewRecord`) can
11169    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11170    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11171    // field's own doc.
11172    let mut nudges: Vec<usize> = vec![0; n];
11173    // The agent now occupying each seat, the ids it has already been through,
11174    // where in the roster the walk began, the class of the last failure, and
11175    // the stem word of a handover not yet asked (full prompt, full budget).
11176    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11177    let mut tried: Vec<BTreeSet<String>> = specs
11178        .iter()
11179        .map(|s| BTreeSet::from([s.id.clone()]))
11180        .collect();
11181    let starts: Vec<usize> = specs
11182        .iter()
11183        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11184        .collect();
11185    let carry = ctx.carry_seats && !roster.is_empty();
11186    // Carried across rounds: ids that failed the seat earlier, and how the
11187    // last failure went (so a repeat of it is not handed over again).
11188    let carried: Vec<BTreeSet<String>> = originals
11189        .iter()
11190        .map(|j| {
11191            state
11192                .seat_history
11193                .get(&j.seat.key)
11194                .filter(|_| carry)
11195                .map(|h| h.failed.clone())
11196                .unwrap_or_default()
11197        })
11198        .collect();
11199    let mut prev: Vec<Option<FailClass>> = originals
11200        .iter()
11201        .map(|j| {
11202            state
11203                .seat_history
11204                .get(&j.seat.key)
11205                .filter(|_| carry)
11206                .and_then(|h| h.last_fail.clone())
11207        })
11208        .collect();
11209    let next_agent =
11210        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11211            let others: BTreeSet<String> = specs
11212                .iter()
11213                .enumerate()
11214                .filter(|(j, _)| *j != i)
11215                .map(|(_, s)| s.id.clone())
11216                .collect();
11217            pick_successor(
11218                roster,
11219                starts[i],
11220                tried,
11221                carry.then(|| &carried[i]),
11222                &others,
11223            )
11224            .cloned()
11225        };
11226    let mut fresh: Vec<Option<String>> = vec![None; n];
11227    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11228    let mut pending: Vec<usize> = (0..n).collect();
11229
11230    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11231    // asks: an agent with a successor is asked once and handed over, and only
11232    // a seat with no successor spends `retries` nudges on the same agent. This
11233    // only guarantees the loop's own termination whatever those say.
11234    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11235    for round in 0..max_rounds {
11236        if pending.is_empty() {
11237            break;
11238        }
11239        let mut batch = Vec::with_capacity(pending.len());
11240        let mut renudged: Vec<&str> = Vec::new();
11241        for &i in &pending {
11242            let src = &originals[i];
11243            // A seat now held by another agent than the job named has no
11244            // session of its own: it gets the full-context prompt whenever
11245            // it is asked in full (the handover ask, a nudge it cannot
11246            // resume).
11247            let full: &str = match &src.handover {
11248                Some(h) if specs[i].id != src.spec.id => h,
11249                _ => &src.prompt,
11250            };
11251            // The prompt and the budget are one decision: a nudge restates
11252            // finished work, a re-sent prompt redoes it.
11253            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11254                (
11255                    full.to_owned(),
11256                    src.timeout,
11257                    format!("{}-{word}-{}", src.stem, specs[i].id),
11258                )
11259            } else if nudges[i] == 0 {
11260                (full.to_owned(), src.timeout, src.stem.clone())
11261            } else {
11262                renudged.push(src.seat.key.as_str());
11263                let why = done[i]
11264                    .as_ref()
11265                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11266                    .unwrap_or_else(|| "no parsable answer".to_owned());
11267                let nudge = prompt::nudge(&why);
11268                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11269                let prompt = if nudged {
11270                    nudge
11271                } else {
11272                    format!("{full}\n\n---\n\n{nudge}")
11273                };
11274                (
11275                    prompt,
11276                    retry_budget(src.timeout, nudged),
11277                    format!("{}-retry{}", src.stem, nudges[i]),
11278                )
11279            };
11280            batch.push(SeatJob {
11281                spec: specs[i].clone(),
11282                seat: seats[i].clone(),
11283                cwd: src.cwd.clone(),
11284                prompt,
11285                timeout,
11286                allow_write: src.allow_write,
11287                sessions: src.sessions,
11288                artifacts: src.artifacts.clone(),
11289                stem,
11290                handover: None,
11291            });
11292        }
11293
11294        if !renudged.is_empty() {
11295            state.event(
11296                ctx.node,
11297                format!("retry {round}: re-asking {}", renudged.join(", ")),
11298            );
11299        }
11300        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11301        let mut still = Vec::new();
11302        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11303            seats[i] = seat;
11304            // No agent was asked: nothing to hand over, nudge, remember or
11305            // charge. The caller sees `resource_block` and stops the node.
11306            if let AgentOutcome::ResourceBlocked(why) = &out {
11307                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11308                continue;
11309            }
11310            let class = FailClass::of(&out);
11311            // A dropped stream is nudged first (the conversation is still
11312            // there to pick up); only a seat whose nudges are spent hands over.
11313            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11314                && nudges[i] < retries
11315                && !roster.is_empty();
11316            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11317                let next = should_hand_over(prev[i].as_ref(), &cur)
11318                    .then(|| next_agent(i, &tried[i], &specs))
11319                    .flatten();
11320                if carry {
11321                    let h = state
11322                        .seat_history
11323                        .entry(originals[i].seat.key.clone())
11324                        .or_default();
11325                    h.failed.insert(specs[i].id.clone());
11326                    h.last_fail = Some(cur.clone());
11327                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11328                        h.last_ok = None;
11329                    }
11330                    // Saved before the next agent is asked, so a restart in
11331                    // between does not forget who failed.
11332                    if let Err(e) = state.save() {
11333                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11334                    }
11335                }
11336                if let Some(next) = next {
11337                    record_handover(
11338                        state,
11339                        ctx.node,
11340                        &originals[i].seat.key,
11341                        &specs[i].id,
11342                        &next.id,
11343                        &cur,
11344                        &fail_reason(&out),
11345                    );
11346                    tried[i].insert(next.id.clone());
11347                    prev[i] = Some(cur.clone());
11348                    seats[i] =
11349                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11350                    specs[i] = next;
11351                    fresh[i] = Some(cur.stem_word().to_owned());
11352                    nudges[i] = 0;
11353                    done[i] = Some(Err(anyhow::anyhow!(
11354                        "handed over after: {}",
11355                        fail_reason(&out)
11356                    )));
11357                    still.push(i);
11358                    continue;
11359                }
11360            }
11361            if carry
11362                && !nudge_first
11363                && let Some(cur) = class.clone()
11364            {
11365                // The chain ended here (no successor, or a repeated failure).
11366                let h = state
11367                    .seat_history
11368                    .entry(originals[i].seat.key.clone())
11369                    .or_default();
11370                h.failed.insert(specs[i].id.clone());
11371                h.last_fail = Some(cur);
11372            }
11373            let parsed = match out {
11374                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11375                AgentOutcome::Quota(o) => {
11376                    last_quota[i] = o.quota.clone();
11377                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11378                }
11379                // Not a parseable answer: the nudge loop re-asks it, which is
11380                // exactly what a dropped stream needs. Just don't hand its raw
11381                // error JSON to `extract_json`.
11382                AgentOutcome::Dropped(o) => {
11383                    let why = o
11384                        .dropped
11385                        .as_ref()
11386                        .map(|d| d.why.as_str())
11387                        .unwrap_or("the CLI ended the stream without delivering its answer");
11388                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11389                }
11390                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11391                    Err(anyhow::anyhow!(e))
11392                }
11393            };
11394            let quota = class == Some(FailClass::Quota);
11395            let failed = parsed.is_err();
11396            if !failed && matches!(ctx.node, "judge" | "vote") {
11397                // A validated answer restores this seat's quorum presence,
11398                // including on the normal path after an all-seat block.
11399                state
11400                    .quota
11401                    .retain(|loss| loss.seat != originals[i].seat.key);
11402            }
11403            done[i] = Some(parsed);
11404            if carry && !failed {
11405                let h = state
11406                    .seat_history
11407                    .entry(originals[i].seat.key.clone())
11408                    .or_default();
11409                h.failed.remove(&specs[i].id);
11410                h.last_ok = Some(specs[i].id.clone());
11411                h.last_fail = None;
11412            }
11413            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11414            // fail the same way; and never re-ask a seat that already parsed.
11415            // A failed agent that still has a successor is not re-asked
11416            // either: the handover was its remedy and has just been refused
11417            // (the chain stops on a repeated failure class). A seat with no
11418            // successor left (a single-agent roster, the roster's tail, or an
11419            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11420            let agent_failure = class.is_some()
11421                && !nudge_first
11422                && !roster.is_empty()
11423                && next_agent(i, &tried[i], &specs).is_some();
11424            if failed && !quota && !agent_failure && nudges[i] < retries {
11425                nudges[i] += 1;
11426                still.push(i);
11427            }
11428        }
11429        pending = still;
11430    }
11431
11432    // One loss per seat whose chain ended on a quota: the intermediate ones
11433    // were absorbed by a handover and are not losses.
11434    for (i, q) in last_quota.into_iter().enumerate() {
11435        if let Some(quota) = q {
11436            losses.push(QuotaLoss::from_report(
11437                originals[i].seat.key.clone(),
11438                ctx.node,
11439                Some(&quota),
11440            ));
11441        }
11442    }
11443
11444    // A stalled quorum already has its own recovery checkpoint. Refresh
11445    // its missing seats before deciding whether it can resume.
11446    if state.status != RunStatus::Stalled
11447        && ctx.node != "deliberate"
11448        && n > 0
11449        && losses.len() == n
11450        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11451    {
11452        block_machine_outage(state, ctx.node, losses);
11453        state.quota.append(losses);
11454    }
11455
11456    seats
11457        .into_iter()
11458        .zip(done)
11459        .zip(nudges)
11460        .map(|((seat, res), attempts)| {
11461            (
11462                seat,
11463                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11464                attempts,
11465            )
11466        })
11467        .collect()
11468}
11469
11470/// A node with no available agent has no task verdict. Preserve the node
11471/// checkpoint and record why a subsequent explicit resume is needed.
11472pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11473    let mut auth: Vec<_> = losses
11474        .iter()
11475        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11476        .map(|loss| loss.agent.clone())
11477        .collect();
11478    auth.sort();
11479    auth.dedup();
11480    let (kind, detail) = if auth.is_empty() {
11481        (
11482            "agent usage limit",
11483            "All seats reached their agent usage limits; resume after usage is available"
11484                .to_owned(),
11485        )
11486    } else {
11487        (
11488            "agent authentication",
11489            format!(
11490                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11491                auth.join(", ")
11492            ),
11493        )
11494    };
11495    state.resource_block = Some(crate::run::ResourceBlock {
11496        node: node.to_owned(),
11497        kind: kind.to_owned(),
11498        cause: crate::run::ResourceBlockCause::Other,
11499        detail: detail.clone(),
11500        at: Timestamp::now(),
11501    });
11502    state.event(node, detail);
11503}
11504
11505/// Did the wave end without asking any agent (see [`block_wave`])?
11506fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11507    results
11508        .iter()
11509        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11510}
11511
11512/// Settle a wave that never spawned anything: every job answers
11513/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11514/// else about the seat (session, history, quota) is touched.
11515fn block_wave(
11516    jobs: Vec<SeatJob>,
11517    state: &mut RunState,
11518    node: &str,
11519    attempt: usize,
11520    why: &str,
11521) -> Vec<(usize, SeatState, AgentOutcome)> {
11522    state.resource_wait = None;
11523    state
11524        .active
11525        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11526    if let Err(e) = state.save() {
11527        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11528    }
11529    jobs.into_iter()
11530        .enumerate()
11531        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11532        .collect()
11533}
11534
11535/// Acquire the shared build cache's lease, waiting out contention within
11536/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11537/// unbounded wait is never acceptable).
11538///
11539/// A first, non-blocking check happens before ever waiting; if it finds the
11540/// lease busy, that fact is logged as a `verify` event *and* flushed with
11541/// [`RunState::save`] immediately — not only once the wait finally succeeds
11542/// or gives up — so a `magi show` run by a different process while this one
11543/// is still waiting reads a `run.json` that says so, rather than whatever it
11544/// looked like before the wait started. The same applies to the terminal
11545/// failure: logged and saved before this returns `Err`, so a caller that
11546/// could not get the lease at all still leaves a legible record of why.
11547async fn acquire_cache_lease(
11548    state: &mut RunState,
11549    cache_dir: &Path,
11550    owner: &crate::cache::Owner,
11551    budget: Duration,
11552    node: &str,
11553    context: &str,
11554    cancel: &(dyn Fn() -> bool + Sync),
11555) -> Result<Option<crate::cache::Guard>> {
11556    let home = crate::run::home();
11557    let started = Instant::now();
11558    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11559        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11560        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11561        Err(e) => {
11562            state.event(
11563                "verify",
11564                format!("{context}: could not check the shared build cache: {e:#}"),
11565            );
11566            if let Err(e2) = state.save() {
11567                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11568            }
11569            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11570        }
11571    };
11572    state.event(
11573        "verify",
11574        format!(
11575            "{context}: waiting for the shared build cache at {} ({})",
11576            cache_dir.display(),
11577            busy.describe()
11578        ),
11579    );
11580    // Seats of this node are waiting, not running: say so, so the countdown a
11581    // reader sees only ever measures an agent that has been spawned.
11582    state.resource_wait = Some(crate::run::ResourceWait {
11583        node: context.to_owned(),
11584        kind: "shared build cache".to_owned(),
11585        holder: busy.describe(),
11586        since: Timestamp::now(),
11587        limit_secs: budget.as_secs(),
11588    });
11589    if let Err(e) = state.save() {
11590        tracing::warn!("could not persist a cache wait: {e:#}");
11591    }
11592    let remaining = budget.saturating_sub(started.elapsed());
11593    let waited = crate::cache::wait_for_unless(
11594        &home,
11595        cache_dir,
11596        owner,
11597        remaining,
11598        Duration::from_secs(5),
11599        cancel,
11600    )
11601    .await;
11602    state.resource_wait = None;
11603    let cause = match &waited {
11604        Ok(Some(_)) => None,
11605        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11606        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11607            Some(crate::run::ResourceBlockCause::Timeout {
11608                waited_secs: started.elapsed().as_secs(),
11609                limit_secs: budget.as_secs(),
11610            })
11611        }
11612        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11613    };
11614    if let Some(cause) = cause {
11615        let detail = match &waited {
11616            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11617            Err(error) => format!("{error:#}"),
11618        };
11619        state.resource_block = Some(crate::run::ResourceBlock {
11620            cause,
11621            node: node.to_owned(),
11622            kind: "shared build cache".into(),
11623            detail,
11624            at: Timestamp::now(),
11625        });
11626        state.event(
11627            "resource",
11628            state.resource_block.as_ref().unwrap().description(),
11629        );
11630    }
11631    if let Err(error) = state.save() {
11632        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11633    }
11634    waited
11635}
11636
11637/// The lease state could not be read at all (unreadable lease file, a
11638/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11639/// known to be contended, so a wave degrades to running without a shared cache
11640/// instead of stopping on a hold that [`crate::triage`] could never release.
11641#[derive(Debug)]
11642struct CacheCheckError(String);
11643
11644impl std::fmt::Display for CacheCheckError {
11645    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11646        f.write_str(&self.0)
11647    }
11648}
11649
11650impl std::error::Error for CacheCheckError {}
11651
11652/// Keep a specific acquisition outcome when the caller adds command evidence.
11653fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11654    state
11655        .resource_block
11656        .get_or_insert_with(|| crate::run::ResourceBlock {
11657            cause: crate::run::ResourceBlockCause::Other,
11658            node: node.to_owned(),
11659            kind: "shared build cache".into(),
11660            detail: format!("{error:#}"),
11661            at: Timestamp::now(),
11662        });
11663    if let Err(error) = state.save() {
11664        tracing::warn!("could not persist cache resource block: {error:#}");
11665    }
11666}
11667
11668/// Run `body` — a verify command batch — while holding the shared build
11669/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11670/// never interleave with another borrower's build against the same
11671/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11672/// timeout, or a human's own `magi review`. See the `cache` module doc for
11673/// why this matters more than Cargo's own per-target locking covers — two
11674/// *different* worktrees building the same package name/version into one
11675/// cache directory is a staleness bug, not a lock contention one.
11676///
11677/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11678/// agent waves do. Repository scheduling also has its own bounded,
11679/// cancellable resource-wait policy. Contention must not consume the
11680/// verification budget: `body` receives the full `budget` after both leases
11681/// are acquired and the cache is fresh. Without a shared cache, only
11682/// repository coordination is acquired before handing over the full budget.
11683///
11684/// A lease that cannot be acquired within the resource wait is reported as a single
11685/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11686/// verification — the same shape a spawn failure already takes in
11687/// [`run_commands`], so a caller need not special-case it.
11688#[allow(clippy::too_many_arguments)]
11689async fn with_cache_lease<'s, F, Fut>(
11690    state: &'s mut RunState,
11691    cache_dir: Option<&Path>,
11692    node: &str,
11693    seat: &str,
11694    worktree: &Path,
11695    head: &str,
11696    budget: Duration,
11697    context: &str,
11698    wait: &crate::repo_lock::WaitPolicy,
11699    body: F,
11700) -> Result<(Vec<CommandOutcome>, bool)>
11701where
11702    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11703    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11704{
11705    // Match editing waves: cache first, repository second. In particular,
11706    // pre_gate must never hold the repository while waiting for a wave's cache.
11707    let started = Instant::now();
11708    let home = crate::run::home();
11709    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11710    let guard = if let Some(cache_dir) = cache_dir {
11711        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11712        match acquire_cache_lease(
11713            state,
11714            cache_dir,
11715            &owner,
11716            resource_limit,
11717            node,
11718            context,
11719            &|| wait.cancelled(),
11720        )
11721        .await
11722        .and_then(|g| {
11723            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11724        }) {
11725            Ok(g) => Some(g),
11726            Err(e) => {
11727                record_cache_error(state, node, &e);
11728                return Ok((
11729                    vec![CommandOutcome {
11730                        command: "(waiting for the shared build cache)".into(),
11731                        code: None,
11732                        output_tail: e.to_string(),
11733                        duration_ms: started.elapsed().as_millis() as u64,
11734                        resource_blocked: true,
11735                    }],
11736                    false,
11737                ));
11738            }
11739        }
11740    } else {
11741        None
11742    };
11743    let _coordination = coordinate_operation(state, worktree, wait).await?;
11744    if let Some(cache_dir) = cache_dir {
11745        let identity = crate::cache::Identity::new(worktree, head);
11746        if let Err(e) = crate::cache::ensure_fresh(
11747            &home,
11748            cache_dir,
11749            &identity,
11750            &state.config.verify.cargo_manifests(),
11751        ) {
11752            record_cache_error(state, node, &e);
11753            state.event(
11754                "verify",
11755                format!(
11756                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11757                    worktree.display(),
11758                    short(head)
11759                ),
11760            );
11761            if let Some(guard) = guard {
11762                guard.release();
11763            }
11764            return Ok((
11765                vec![CommandOutcome {
11766                    command: "(confirming the shared build cache is fresh)".into(),
11767                    code: None,
11768                    output_tail: e.to_string(),
11769                    duration_ms: started.elapsed().as_millis() as u64,
11770                    resource_blocked: true,
11771                }],
11772                false,
11773            ));
11774        }
11775    }
11776    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11777    if !timed_out_pids.is_empty() {
11778        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11779    }
11780    // Drop repository coordination before releasing the cache, reversing the
11781    // acquisition order. A cancelled/error future also drops both guards.
11782    drop(_coordination);
11783    if let Some(guard) = guard {
11784        guard.release();
11785    }
11786    Ok((outcomes, retried))
11787}
11788
11789/// Poll `pids` — commands [`run_commands`] reports as still running when its
11790/// own timeout elapsed — until every one is confirmed gone, or
11791/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11792///
11793/// Real confirmation where confirmation is possible, not a substitute for
11794/// full process-tree observation: a grandchild the timed-out process spawned
11795/// and that survives independently of it is invisible to a pid check the
11796/// same way it always was, and continuing to observe and collect *that*
11797/// stays a different piece of work with its own owner. This only narrows a
11798/// fixed blind wait into an actual check of the pids this process does know
11799/// about.
11800async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11801    wait_for_pids_with(
11802        pids,
11803        crate::proc::pid_alive,
11804        LEASE_RELEASE_POLL,
11805        LEASE_RELEASE_MAX_WAIT,
11806    )
11807    .await;
11808}
11809
11810/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11811/// interval and ceiling supplied by the caller, so the polling *logic* -
11812/// returns as soon as every pid reports dead, gives up at the ceiling
11813/// otherwise - is testable on millisecond durations without asking the real
11814/// OS about a pid at all.
11815async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11816    pids: &[u32],
11817    alive: F,
11818    poll: Duration,
11819    max_wait: Duration,
11820) {
11821    let deadline = Instant::now() + max_wait;
11822    loop {
11823        if pids.iter().all(|&pid| !alive(pid)) {
11824            return;
11825        }
11826        if Instant::now() >= deadline {
11827            return;
11828        }
11829        tokio::time::sleep(poll).await;
11830    }
11831}
11832
11833/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11834/// admission that it could not even get a verify command to run, as opposed
11835/// to evidence the command actually produced? A caller that would otherwise
11836/// read a resource-blocked outcome as a red command must check this first:
11837/// see [`Runner::gate`], which retries rather than records `Blocked` when
11838/// this is true.
11839fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11840    outcomes.iter().any(|o| o.resource_blocked)
11841}
11842
11843/// What [`Runner::gate_fix_round`] decided.
11844pub(crate) enum GateFix {
11845    /// The tree changed and `verify.e2e` is still green: run the gate again.
11846    Retry,
11847    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11848    /// last failure stands and the run ends blocked.
11849    Stop,
11850    /// `verify.e2e` could not run after the fix (magi's own contention):
11851    /// decide nothing now, a later reentry retries.
11852    Defer,
11853}
11854
11855/// Is every red command in `outcomes` an ordinary failure the code could
11856/// explain: it ran, exited non-zero, and said something?
11857///
11858/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11859/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11860/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11861/// what the command is stays the gate's business.
11862pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11863    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11864    red.peek().is_some()
11865        && red.all(|o| {
11866            !o.resource_blocked
11867                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11868                && !o.output_tail.trim().is_empty()
11869        })
11870}
11871
11872/// Describe one verify command's outcome for the event log, distinguishing a
11873/// build/link failure — the toolchain never produced a binary to run — from
11874/// an actual test failure, since only the latter is a verdict on the patch.
11875fn e2e_outcome_label(o: &CommandOutcome) -> String {
11876    if o.ok() {
11877        return "pass".to_owned();
11878    }
11879    let reason = if o.build_failed() {
11880        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11881    } else {
11882        format!("FAIL ({:?})", o.code)
11883    };
11884    format!(
11885        "{reason}\n{}",
11886        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11887    )
11888}
11889
11890/// Run `verify.e2e`, retrying once if the first attempt could not build or
11891/// link — a build/link failure is frequently a race against a shared
11892/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11893/// `verify` event per command, tagged with `context` (normally `"round N"`)
11894/// so the two call sites that need this — the ordinary per-round leg in
11895/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11896/// it will ever call a round green — read identically in the event log.
11897async fn run_e2e_with_retry(
11898    state: &mut RunState,
11899    shell: &[String],
11900    commands: &[String],
11901    worktree: &Path,
11902    timeout: Duration,
11903    context: &str,
11904) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11905    let (mut e2e, mut timed_out_pids) = run_commands(
11906        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11907    )
11908    .await;
11909    for o in &e2e {
11910        state.event(
11911            "verify",
11912            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11913        );
11914    }
11915    // A build/link failure is not a verdict on the patch — it is frequently a
11916    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11917    // one retry before letting a red like that decide the round.
11918    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11919    if verify_retried {
11920        state.event(
11921            "verify",
11922            format!(
11923                "{context}: verify could not build/link, not a test result — retrying once \
11924                 before concluding"
11925            ),
11926        );
11927        let retried = run_commands(
11928            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11929        )
11930        .await;
11931        e2e = retried.0;
11932        // Both attempts' timeouts matter, not just the last one: the first
11933        // attempt's descendants may still be alive alongside the retry's.
11934        timed_out_pids.extend(retried.1);
11935        for o in &e2e {
11936            state.event(
11937                "verify",
11938                format!(
11939                    "{context}: retry `{}` -> {}",
11940                    o.command,
11941                    e2e_outcome_label(o)
11942                ),
11943            );
11944        }
11945    }
11946    (e2e, verify_retried, timed_out_pids)
11947}
11948
11949/// Run configured shell commands in `cwd`, in order. The second element is
11950/// the pid of every command that hit `timeout` and was still running when
11951/// this stopped waiting on it (best-effort: `None` when the platform did not
11952/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11953/// that releases a shared resource afterward needs to know.
11954///
11955/// Records `task` into [`RunState::active`] at every command boundary
11956/// (`RunState::task_command`) and clears it once the whole list has run
11957/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11958/// for minutes with no seat and no output of its own to show for it (see
11959/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11960/// apart from "not yet run" without this), and this is the only place that
11961/// knows which command is running right now and how many are left. Three
11962/// saves per command — start, not per second — matching the same "only at a
11963/// boundary" rule [`wave`] already follows for seats.
11964#[allow(clippy::too_many_arguments)]
11965async fn run_commands(
11966    state: &mut RunState,
11967    node: &str,
11968    task: &str,
11969    attempt: usize,
11970    shell: &[String],
11971    commands: &[String],
11972    cwd: &Path,
11973    timeout: Duration,
11974) -> (Vec<CommandOutcome>, Vec<u32>) {
11975    if commands.is_empty() {
11976        // Nothing to mark as running and nothing to clear — an empty list
11977        // means "not configured", and touching `active` (or the disk) over
11978        // that would be a write for every round of a repo with no
11979        // `verify.e2e` / `verify.gate` commands at all.
11980        return (Vec::new(), Vec::new());
11981    }
11982    let mut out = Vec::new();
11983    let mut timed_out_pids = Vec::new();
11984    let total = commands.len();
11985    for (idx, command) in commands.iter().enumerate() {
11986        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11987        if let Err(e) = state.save() {
11988            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11989        }
11990        let started = Instant::now();
11991        let mut cmd = tokio::process::Command::new(&shell[0]);
11992        cmd.quiet();
11993        // Verify the checkout as a standalone caller. Worker ownership and
11994        // pinned executable selection belong to this driver, not to tests or
11995        // tools it runs. Preserve home/seat context and Running's containment.
11996        for key in crate::worker::INVOCATION_ENV {
11997            cmd.env_remove(key);
11998        }
11999        cmd.args(&shell[1..])
12000            .arg(command)
12001            .current_dir(cwd)
12002            .stdin(std::process::Stdio::null())
12003            .stdout(std::process::Stdio::piped())
12004            .stderr(std::process::Stdio::piped())
12005            .kill_on_drop(true);
12006        let mut outputs = Vec::new();
12007        let protected = (|| -> Result<()> {
12008            crate::incremental::configure(&mut cmd, cwd)?;
12009            let mut paths = vec![cwd.to_owned()];
12010            if let Some(cache) = state.config.cache_dir() {
12011                paths.push(cache);
12012            }
12013            outputs.extend(crate::retention::OutputLease::admit(
12014                &crate::run::home(),
12015                &paths,
12016                state.config.disk.min_free_bytes,
12017            )?);
12018            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12019                outputs.push(guard);
12020            }
12021            if let Some(cache) = state.config.cache_dir() {
12022                outputs.push(crate::retention::OutputLease::cache(
12023                    &crate::run::home(),
12024                    &cache,
12025                    false,
12026                )?);
12027            }
12028            Ok(())
12029        })();
12030        if let Err(e) = protected {
12031            let detail = format!("execution admission unavailable: {e:#}");
12032            state.resource_block = Some(crate::run::ResourceBlock {
12033                cause: crate::run::ResourceBlockCause::Other,
12034                node: node.to_owned(),
12035                kind: if detail.contains("disk admission") {
12036                    "disk admission"
12037                } else {
12038                    "output ownership"
12039                }
12040                .to_owned(),
12041                detail: detail.clone(),
12042                at: Timestamp::now(),
12043            });
12044            out.push(CommandOutcome {
12045                command: command.clone(),
12046                code: None,
12047                output_tail: detail,
12048                duration_ms: 0,
12049                resource_blocked: true,
12050            });
12051            break;
12052        }
12053        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12054        let (code, body) = match spawned {
12055            Ok(mut running) => {
12056                let pid = running.child().id();
12057                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12058                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12059                let mut paths = vec![cwd.to_owned()];
12060                if let Some(cache) = state.config.cache_dir() {
12061                    paths.push(cache);
12062                }
12063                let mut pressure = None;
12064                let status = tokio::select! {
12065                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12066                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12067                        pressure = Some(reason); None
12068                    }
12069                };
12070                let (code, failure) = match status {
12071                    Some(Ok(Ok(status))) => (
12072                        status.code(),
12073                        status
12074                            .code()
12075                            .is_none()
12076                            .then(|| format!("command terminated: {status}")),
12077                    ),
12078                    Some(Ok(Err(error))) => {
12079                        (None, Some(format!("failed waiting for command: {error}")))
12080                    }
12081                    Some(Err(_)) => {
12082                        // Leased cleanup confirms process-tree death itself.
12083                        // Unleased commands still need the caller's PID check
12084                        // after kill-on-drop, before releasing cache ownership.
12085                        if !running.coordinated()
12086                            && let Some(pid) = pid
12087                        {
12088                            timed_out_pids.push(pid);
12089                        }
12090                        (
12091                            None,
12092                            Some(format!("timed out after {}s", timeout.as_secs())),
12093                        )
12094                    }
12095                    None => (None, None),
12096                };
12097                running.finish().await;
12098                if let Some(reason) = pressure {
12099                    state.resource_block = Some(crate::run::ResourceBlock {
12100                        cause: crate::run::ResourceBlockCause::Other,
12101                        node: node.to_owned(),
12102                        kind: "disk pressure".to_owned(),
12103                        detail: reason.clone(),
12104                        at: Timestamp::now(),
12105                    });
12106                    out.push(CommandOutcome {
12107                        command: command.clone(),
12108                        code: None,
12109                        output_tail: reason,
12110                        duration_ms: started.elapsed().as_millis() as u64,
12111                        resource_blocked: true,
12112                    });
12113                    break;
12114                }
12115                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12116                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12117                if let Some(failure) = failure {
12118                    body.push('\n');
12119                    body.push_str(&failure);
12120                }
12121                (code, body)
12122            }
12123            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12124        };
12125        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12126        if !body.is_empty() {
12127            // The full text is kept whatever the summary dropped. A fresh
12128            // name per command so a resume or retry never overwrites an
12129            // earlier run's output. A failed write changes no verdict.
12130            match crate::run::write_verification_artifact(state, task, &body) {
12131                Ok(name) if summary != body => {
12132                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12133                }
12134                Ok(_) => {}
12135                Err(e) => {
12136                    let warning = format!("could not save the full {task} output: {e:#}");
12137                    tracing::warn!("{warning}");
12138                    state.event(node, warning);
12139                    if summary != body {
12140                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12141                    }
12142                }
12143            }
12144        }
12145        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12146        out.push(CommandOutcome {
12147            command: command.clone(),
12148            code,
12149            output_tail: summary,
12150            duration_ms: started.elapsed().as_millis() as u64,
12151            resource_blocked,
12152        });
12153        if resource_blocked {
12154            state.resource_block = Some(crate::run::ResourceBlock {
12155                cause: crate::run::ResourceBlockCause::Other,
12156                node: node.to_owned(),
12157                kind: "incremental cache admission".into(),
12158                detail: body,
12159                at: Timestamp::now(),
12160            });
12161            break;
12162        }
12163    }
12164    state.task_finished(task);
12165    if let Err(e) = state.save() {
12166        tracing::warn!("could not persist the end of {task}: {e:#}");
12167    }
12168    (out, timed_out_pids)
12169}
12170
12171/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12172/// section (`report::run`) and in the `merge` event this node records — for
12173/// the operator to run by hand.
12174///
12175/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12176/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12177/// merge commits", or "require linear history") rejects the push a `--no-ff`
12178/// merge would produce, which is exactly the guidance this function replaces.
12179/// `message`'s first line becomes the squash commit's subject, matching the
12180/// note `report::run` prints alongside this command — see that function for
12181/// why an explicit subject is not optional there.
12182fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12183    let repo = repo.display();
12184    match style {
12185        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12186        MergeStyle::Squash => {
12187            // The subject sits inside double quotes, and a title an agent
12188            // wrote may carry the characters that break out of them.
12189            let subject = message
12190                .lines()
12191                .next()
12192                .unwrap_or(branch)
12193                .replace(['\\', '"', '$', '`'], "");
12194            format!(
12195                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12196            )
12197        }
12198        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12199    }
12200}
12201
12202/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12203/// under the hood, rejects a `title` over 256 characters and the whole
12204/// command fails — no PR at all, for a run whose body was otherwise fine
12205/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12206/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12207/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12208/// this bound. It is a margin, not a guarantee — a title packed
12209/// with multi-unit characters could still in principle land close to the
12210/// edge, but a real task title's occasional emoji or accented letter fits
12211/// comfortably inside it.
12212const PR_TITLE_MAX: usize = 240;
12213
12214/// A pull request title from the opening line of `text`, or `None` when that
12215/// line is not English (GitHub text is) or has no letters.
12216///
12217/// A line within `max` is kept as is. A longer one is cut at the end of its
12218/// first sentence when that falls inside `max`, else at a word boundary with a
12219/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12220/// make every merely-truncated title look non-English). The language check
12221/// runs on the kept text before any mark is added, so only what GitHub will
12222/// show is judged: an English opening followed by non-ASCII far past the cut
12223/// still passes.
12224fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12225    let line = queue::first_line(text)?;
12226    let chars: Vec<char> = line.chars().collect();
12227    let (kept, mark) = if chars.len() <= max {
12228        (line.to_owned(), "")
12229    } else if let Some(end) = sentence_end(&chars, max) {
12230        (chars[..end].iter().collect::<String>(), "")
12231    } else {
12232        let room = max.saturating_sub(3);
12233        // Cut at the last space inside the room; when the char just past the
12234        // room is a space the room already ends on a word.
12235        let cut = if chars[room].is_whitespace() {
12236            room
12237        } else {
12238            chars[..room]
12239                .iter()
12240                .rposition(|c| c.is_whitespace())
12241                .unwrap_or(room)
12242        };
12243        let head: String = chars[..cut].iter().collect();
12244        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12245        (head.to_owned(), "...")
12246    };
12247    if kept.is_empty() {
12248        return None;
12249    }
12250    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12251        return None;
12252    }
12253    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12254        return None;
12255    }
12256    Some(format!("{kept}{mark}"))
12257}
12258
12259/// The char length of the first sentence of `chars` when it ends within `max`
12260/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12261/// abbreviations so `e.g. foo` does not end a title early.
12262fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12263    const MIN: usize = 20;
12264    for i in MIN..max.min(chars.len()) {
12265        if !matches!(chars[i], '.' | '!' | '?') {
12266            continue;
12267        }
12268        let Some(&next) = chars.get(i + 1) else {
12269            continue;
12270        };
12271        if !next.is_whitespace() {
12272            continue;
12273        }
12274        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12275        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12276            continue;
12277        }
12278        let word: String = chars[..i]
12279            .iter()
12280            .rev()
12281            .take_while(|c| !c.is_whitespace())
12282            .collect::<Vec<_>>()
12283            .into_iter()
12284            .rev()
12285            .collect();
12286        let word = word.to_ascii_lowercase();
12287        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12288            continue;
12289        }
12290        let end = chars[..i]
12291            .iter()
12292            .rposition(|c| !c.is_whitespace())
12293            .map_or(i, |p| p + 1);
12294        return Some(end);
12295    }
12296    None
12297}
12298
12299/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12300/// change: a title and a body describing what was *implemented*, not the task
12301/// that asked for it. A task reads as a request; a reader of the merged
12302/// history wants the change.
12303struct PrMessage {
12304    title: String,
12305    body: String,
12306}
12307
12308impl PrMessage {
12309    /// Title, blank line, body. The first line is the squash/merge commit
12310    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12311    /// has to stay one sensible line.
12312    fn commit_message(&self) -> String {
12313        format!("{}\n\n{}", self.title, self.body)
12314    }
12315}
12316
12317/// The text after a leading `TITLE:` (any case) on `line`.
12318fn title_marker(line: &str) -> Option<&str> {
12319    let line = line.trim();
12320    let head = line.get(..6)?;
12321    head.eq_ignore_ascii_case("title:")
12322        .then(|| line[6..].trim())
12323}
12324
12325/// The implementer's own one-line title: the `TITLE:` line the implement
12326/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12327/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12328/// source, and a title that says as much is refused here too.
12329fn summary_title(summary: &str) -> Option<String> {
12330    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12331    let raw = title_marker(first)?;
12332    if raw.is_empty() {
12333        return None;
12334    }
12335    let title = queue::title_from(raw, PR_TITLE_MAX);
12336    let lower = title.to_ascii_lowercase();
12337    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12338        return None;
12339    }
12340    Some(title)
12341}
12342
12343/// How `open_review`'s instruction begins; see [`landing_title`].
12344const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12345
12346/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12347pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12348
12349/// Does this run review work that already existed, rather than implement a
12350/// task? Runs recorded before `reviewed_commits` existed carry only the
12351/// candidate marker.
12352pub(crate) fn is_review_run(state: &RunState) -> bool {
12353    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12354}
12355
12356/// The title of a review-only run: the subject of the oldest commit under
12357/// review. Later commits are usually fixups, and `instruction` is the review
12358/// prompt, which says nothing about the change. GitHub text is English, so a
12359/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12360fn review_title(state: &RunState) -> Option<String> {
12361    english_subject(
12362        state.reviewed_commits.as_ref()?.first()?,
12363        state.config.graph.github_english(),
12364    )
12365}
12366
12367/// `raw` as a pull request title, or `None` when it is blank, not English
12368/// (GitHub text is), or one of magi's own candidate commit subjects.
12369fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12370    let raw = raw.trim();
12371    if raw.is_empty() {
12372        return None;
12373    }
12374    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12375        return None;
12376    }
12377    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12378        return None;
12379    }
12380    let title = queue::title_from(raw, PR_TITLE_MAX);
12381    let lower = title.to_ascii_lowercase();
12382    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12383        return None;
12384    }
12385    Some(title)
12386}
12387
12388/// What a review-only run's branch says about itself, read at the moment the
12389/// pull request is opened.
12390#[derive(Debug, Clone, PartialEq, Eq)]
12391struct BranchFacts {
12392    /// `(subject, body)` of each commit, oldest first.
12393    commits: Vec<(String, String)>,
12394    /// Trimmed `git diff --stat`.
12395    stat: String,
12396}
12397
12398/// Longest diff stat shown: this many file lines plus the summary line.
12399const STAT_FILE_LINES: usize = 25;
12400/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12401const COMMITS_MAX_CHARS: usize = 20_000;
12402
12403/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12404/// say or finds nothing, so the caller falls back to what the run recorded.
12405async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12406    let commits = git::commit_log(repo, base, branch).await.ok()?;
12407    if commits.is_empty() {
12408        return None;
12409    }
12410    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12411    let lines: Vec<&str> = stat.lines().collect();
12412    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12413        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12414        let more = format!(" ... {omitted} more file(s)");
12415        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12416        kept.push(&more);
12417        kept.push(lines[lines.len() - 1]);
12418        kept.join("\n")
12419    } else {
12420        lines.join("\n")
12421    };
12422    Some(BranchFacts { commits, stat })
12423}
12424
12425/// Defang what would break the surrounding markdown: a closing `</details>`
12426/// and a code fence.
12427fn markdown_safe(text: &str) -> String {
12428    text.replace("</details>", "&lt;/details&gt;")
12429        .replace("\x60\x60\x60", "~~~")
12430}
12431
12432fn neutral_title(state: &RunState, winner: char) -> String {
12433    format!(
12434        "chore: land candidate {} of run {}",
12435        winner.to_ascii_uppercase(),
12436        state.id
12437    )
12438}
12439
12440/// Reuse the graph's bounded fixer without starting another graph or driver.
12441/// Copy the working state back even on errors so spent rounds remain visible.
12442pub(crate) async fn repair_landing_gate(
12443    state: &mut RunState,
12444    winner: &Candidate,
12445    outcomes: &[CommandOutcome],
12446) -> Result<GateFix> {
12447    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12448        return Ok(GateFix::Stop);
12449    }
12450    let mut runner = Runner {
12451        roles: state.config.resolve_roles()?,
12452        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12453        state: state.clone(),
12454        pause: Pause::new(),
12455        interrupt: Pause::new(),
12456    };
12457    let result = runner.gate_fix_round(winner, outcomes).await;
12458    *state = runner.state;
12459    result
12460}
12461
12462/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12463/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12464pub(crate) async fn verify_current_head(
12465    state: &mut RunState,
12466    winner: &Candidate,
12467    wait: &crate::repo_lock::WaitPolicy,
12468) -> Result<bool> {
12469    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12470    // Preserve the previous record before replacing it, including on crash recovery.
12471    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12472        state
12473            .verification_history
12474            .push(crate::run::VerificationEvidence {
12475                superseded_by: Some(head.clone()),
12476                archived_at: Some(Timestamp::now()),
12477                fixes_spent: Some(state.gate_fixes.len()),
12478                gate_head: None,
12479                gate_ran: false,
12480                gate: Vec::new(),
12481                e2e: state.head_e2e.take(),
12482            });
12483    }
12484    if let Some(e) = &state.head_e2e
12485        && e.head == head
12486    {
12487        if e.passed {
12488            return Ok(true);
12489        }
12490        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12491            state.status = RunStatus::Blocked;
12492            state.save()?;
12493            return Ok(false);
12494        }
12495    } else if state.reviews.last().is_some_and(|r| {
12496        r.verified_head.as_deref() == Some(head.as_str())
12497            && (r.e2e_status() == E2eStatus::Passed
12498                || (r.e2e_status() == E2eStatus::NotConfigured
12499                    && state.config.verify.e2e.is_empty()))
12500    }) || state
12501        .gate_repair_e2e
12502        .as_ref()
12503        .is_some_and(|e| e.head == head && e.passed)
12504    {
12505        return Ok(true);
12506    }
12507    if let Some(e) = &state.gate_repair_e2e
12508        && e.head == head
12509        && !e.passed
12510        && !e.outcomes.is_empty()
12511        && !verify_inconclusive(&e.outcomes)
12512    {
12513        state.status = RunStatus::Blocked;
12514        state.save()?;
12515        return Ok(false);
12516    }
12517    state.head_e2e = Some(crate::run::GateRepairE2e {
12518        head: head.clone(),
12519        passed: false,
12520        outcomes: Vec::new(),
12521    });
12522    state.save()?;
12523    let commands = state.config.verify.e2e.clone();
12524    let shell = state.config.shell();
12525    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12526    let cache_dir = state.config.cache_dir();
12527    let outcomes = if commands.is_empty() {
12528        Vec::new()
12529    } else {
12530        let (outcomes, _) = with_cache_lease(
12531            state,
12532            cache_dir.as_deref(),
12533            "e2e",
12534            "e2e",
12535            &winner.worktree,
12536            &head,
12537            timeout,
12538            "current head e2e",
12539            wait,
12540            |state, budget| async move {
12541                run_e2e_with_retry(
12542                    state,
12543                    &shell,
12544                    &commands,
12545                    &winner.worktree,
12546                    budget,
12547                    "current head e2e",
12548                )
12549                .await
12550            },
12551        )
12552        .await?;
12553        outcomes
12554    };
12555    let unresolved = verify_inconclusive(&outcomes);
12556    let passed = !unresolved
12557        && outcomes.iter().all(CommandOutcome::ok)
12558        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12559    if !outcomes.is_empty() && !unresolved {
12560        state.record_base_progress(&head);
12561    }
12562    state.head_e2e = Some(crate::run::GateRepairE2e {
12563        head: head.clone(),
12564        passed,
12565        outcomes,
12566    });
12567    if !passed && !unresolved {
12568        state.status = RunStatus::Blocked;
12569        state.event(
12570            "e2e",
12571            format!("verify.e2e failed on head {head}; nothing published"),
12572        );
12573    }
12574    state.save()?;
12575    Ok(passed)
12576}
12577
12578/// Recheck a repair's e2e requirement across failures, resource deferrals and
12579/// process interruptions. A successful result only applies to its exact SHA.
12580pub(crate) async fn check_gate_repair_e2e(
12581    state: &mut RunState,
12582    winner: &Candidate,
12583    wait: &crate::repo_lock::WaitPolicy,
12584) -> Result<GateFix> {
12585    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12586        return Ok(GateFix::Retry);
12587    }
12588    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12589    if state
12590        .gate_repair_e2e
12591        .as_ref()
12592        .is_some_and(|r| r.head == head && r.passed)
12593    {
12594        return Ok(GateFix::Retry);
12595    }
12596    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12597        head: head.clone(),
12598        passed: false,
12599        outcomes: Vec::new(),
12600    });
12601    state.save()?;
12602    let commands = state.config.verify.e2e.clone();
12603    let outcomes = if commands.is_empty() {
12604        Vec::new()
12605    } else {
12606        let shell = state.config.shell();
12607        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12608        let cache_dir = state.config.cache_dir();
12609        let (outcomes, _) = with_cache_lease(
12610            state,
12611            cache_dir.as_deref(),
12612            "e2e",
12613            "e2e",
12614            &winner.worktree,
12615            &head,
12616            timeout,
12617            "gate repair e2e",
12618            wait,
12619            |state, budget| async move {
12620                run_e2e_with_retry(
12621                    state,
12622                    &shell,
12623                    &commands,
12624                    &winner.worktree,
12625                    budget,
12626                    "gate repair e2e",
12627                )
12628                .await
12629            },
12630        )
12631        .await?;
12632        outcomes
12633    };
12634    let deferred = verify_inconclusive(&outcomes);
12635    let passed = outcomes.iter().all(CommandOutcome::ok)
12636        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12637    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12638        head,
12639        passed,
12640        outcomes,
12641    });
12642    state.save()?;
12643    if deferred {
12644        Ok(GateFix::Defer)
12645    } else if !passed {
12646        state.event(
12647            "gate",
12648            "verify.e2e failed after gate repair; nothing published",
12649        );
12650        Ok(GateFix::Stop)
12651    } else {
12652        Ok(GateFix::Retry)
12653    }
12654}
12655
12656/// Run the same configured gate for a landing head and a fresh graph.
12657pub(crate) async fn run_landing_gate(
12658    state: &mut RunState,
12659    winner: &Candidate,
12660    wait: &crate::repo_lock::WaitPolicy,
12661) -> Result<Vec<CommandOutcome>> {
12662    let shell = state.config.shell();
12663    let gate_commands = state.config.verify.gate.clone();
12664    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12665    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12666        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12667        Vec::new()
12668    } else {
12669        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12670        let cache_dir = state.config.cache_dir();
12671        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12672        let (outcomes, _) = with_cache_lease(
12673            state,
12674            cache_dir.as_deref(),
12675            "gate",
12676            "gate",
12677            &winner.worktree,
12678            &head,
12679            timeout,
12680            "final gate",
12681            wait,
12682            |state, budget| {
12683                let shell = shell.clone();
12684                let gate_commands = gate_commands.clone();
12685                let winner = winner.clone();
12686                async move {
12687                    Runner::run_pre_gate(state, &winner).await;
12688                    // Bind command evidence after mechanical fixes, before commands start.
12689                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12690                    let (outcomes, timed_out_pids) = run_commands(
12691                        state,
12692                        "gate",
12693                        "gate",
12694                        0,
12695                        &shell,
12696                        &gate_commands,
12697                        &winner.worktree,
12698                        budget,
12699                    )
12700                    .await;
12701                    (outcomes, false, timed_out_pids)
12702                }
12703            },
12704        )
12705        .await?;
12706        outcomes
12707    };
12708    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12709    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12710        let mut outcomes = outcomes;
12711        outcomes.push(CommandOutcome {
12712            command: "gate head stability".into(), code: None,
12713            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12714            duration_ms: 0, resource_blocked: false,
12715        });
12716        return Ok(outcomes);
12717    }
12718    if outcomes.is_empty() {
12719        // Nothing configured to check — distinct from every other
12720        // silence in this run's event log, since an empty `gate` alone
12721        // no longer says whether the gate ran at all (see
12722        // `RunState::gate_ran`'s own doc).
12723        state.event("gate", "no gate commands configured; not tested");
12724    }
12725    for o in &outcomes {
12726        state.event(
12727            "gate",
12728            format!(
12729                "`{}` -> {}",
12730                o.command,
12731                if o.ok() {
12732                    "pass".to_owned()
12733                } else {
12734                    format!(
12735                        "FAIL ({:?})\n{}",
12736                        o.code,
12737                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12738                    )
12739                }
12740            ),
12741        );
12742    }
12743    Ok(outcomes)
12744}
12745
12746/// The pull request title to hand to `land::merge_subject`. A review-only run
12747/// opened by an earlier build titled its pull request with the review prompt;
12748/// that title is dropped (empty, so the fallback applies) rather than landed.
12749/// Any other title, including an operator's rename, passes through untouched,
12750/// and so does every title of a run that implements a task.
12751pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12752    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12753        ""
12754    } else {
12755        pr_title
12756    }
12757}
12758
12759/// What the squash subject falls back to when the pull request title is empty
12760/// or candidate-shaped: for a review-only run the derived title, never the
12761/// review prompt held in `instruction`.
12762pub fn landing_subject_source(state: &RunState) -> String {
12763    if is_review_run(state) {
12764        let winner = state.candidates.first().map_or('A', |c| c.label);
12765        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12766    }
12767    state.instruction.clone()
12768}
12769
12770/// `summary` without its `TITLE:` line, which the pull request title already
12771/// carries.
12772fn summary_without_title(summary: &str) -> String {
12773    let mut lines = summary.trim().lines().peekable();
12774    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12775        lines.next();
12776    }
12777    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12778}
12779
12780/// The pull request title and body for the winning candidate.
12781///
12782/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12783/// the task's own opening line via [`queue::title_from`] when there is none.
12784/// `state.instruction` can open with blank lines (`task_text` only rejects a
12785/// body that is blank *entirely*), which `title_from` skips.
12786///
12787/// Publishing uses an independent description generated from the final diff
12788/// and structured evidence. Internal summaries and reports remain local.
12789/// Non-publishing previews may display the original task and candidate summary.
12790/// The footer retains the run and candidate provenance.
12791#[cfg(test)]
12792fn pr_message(state: &RunState, winner: char) -> PrMessage {
12793    pr_message_with(state, winner, None)
12794}
12795
12796/// [`pr_message`] with what the branch of a review-only run says about itself.
12797/// `facts` is ignored for a run that implements a task.
12798#[cfg(test)]
12799fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12800    let raw = pr_message_raw(
12801        state,
12802        winner,
12803        facts,
12804        crate::pr_outcome::FinalHead::Unreported,
12805        ChangeNote::NotAsked,
12806    );
12807    let id = crate::scrub::Identity::current();
12808    PrMessage {
12809        title: crate::scrub::scrub(&raw.title, &id),
12810        body: crate::scrub::scrub(&raw.body, &id),
12811    }
12812}
12813
12814/// What the public description may say about the change itself.
12815#[derive(Debug, Clone, Copy)]
12816enum ChangeNote<'a> {
12817    /// A local preview or non-publishing merge mode: no public generation needed.
12818    NotAsked,
12819    /// A summary written from the final diff.
12820    Summary(&'a crate::run::ChangeSummary),
12821    /// A summary was asked for and could not be produced; the text says why.
12822    Unavailable(&'a str),
12823}
12824
12825fn pr_message_raw(
12826    state: &RunState,
12827    winner: char,
12828    facts: Option<&BranchFacts>,
12829    head: crate::pr_outcome::FinalHead<'_>,
12830    change: ChangeNote<'_>,
12831) -> PrMessage {
12832    if !matches!(change, ChangeNote::NotAsked) {
12833        let (title, mut body) = match change {
12834            ChangeNote::Summary(c) => (
12835                c.title.clone(),
12836                format!("## Summary\n\n{}\n\n", c.body.trim()),
12837            ),
12838            ChangeNote::Unavailable(why) => (
12839                neutral_title(state, winner),
12840                format!(
12841                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12842                ),
12843            ),
12844            ChangeNote::NotAsked => unreachable!(),
12845        };
12846        body.push_str(&public_pr_evidence(state, head));
12847        body.push_str(&format!(
12848            "\n---\nmagi:run/{} magi:candidate-{}\n",
12849            state.id,
12850            winner.to_ascii_lowercase()
12851        ));
12852        return PrMessage { title, body };
12853    }
12854    pr_preview(state, winner, facts, head)
12855}
12856
12857/// Append public facts from the run records after every prose generation,
12858/// including posting-gate rewrites. The agent cannot drop this provenance.
12859fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12860    let mut body = crate::pr_outcome::review_rounds(state);
12861    body.push_str(&crate::pr_outcome::final_verification(state, head));
12862    let open = state.last_round_findings();
12863    if !open.is_empty() {
12864        body.push_str("## Open review findings\n\n");
12865        for f in open {
12866            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12867        }
12868        body.push('\n');
12869    }
12870    if let Some(c) = state.change_summary.as_ref()
12871        && head == crate::pr_outcome::FinalHead::At(&c.head)
12872    {
12873        body.push_str(&format!(
12874            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12875            c.base, c.head
12876        ));
12877    }
12878    body
12879}
12880
12881/// Local-only preview of the operator's task and candidate summary.
12882fn pr_preview(
12883    state: &RunState,
12884    winner: char,
12885    facts: Option<&BranchFacts>,
12886    head: crate::pr_outcome::FinalHead<'_>,
12887) -> PrMessage {
12888    let summary = state
12889        .candidates
12890        .iter()
12891        .find(|c| c.label == winner)
12892        .map(|c| c.summary.as_str())
12893        .unwrap_or_default();
12894    // The fallback is the operator's own words and may not be English; GitHub
12895    // text always is, so a non-English task gets a neutral title instead.
12896    let review = is_review_run(state);
12897    let title = if review {
12898        facts
12899            .and_then(|f| {
12900                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12901            })
12902            .or_else(|| review_title(state))
12903            .or_else(|| {
12904                state
12905                    .candidates
12906                    .iter()
12907                    .find(|c| c.label == winner)
12908                    .filter(|c| !c.branch.starts_with("magi/"))
12909                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12910            })
12911            .unwrap_or_else(|| neutral_title(state, winner))
12912    } else {
12913        summary_title(summary).unwrap_or_else(|| {
12914            english_title(
12915                &state.instruction,
12916                PR_TITLE_MAX,
12917                state.config.graph.github_english(),
12918            )
12919            .unwrap_or_else(|| neutral_title(state, winner))
12920        })
12921    };
12922
12923    let mut body = String::new();
12924    let what = summary_without_title(summary);
12925    if !what.is_empty() {
12926        body.push_str("## Summary\n\n");
12927        body.push_str(&what);
12928        body.push_str("\n\n");
12929    }
12930
12931    // Review history uses structured facts across all rounds, never reports.
12932    body.push_str(&crate::pr_outcome::review_rounds(state));
12933    body.push_str(&crate::pr_outcome::final_verification(state, head));
12934    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12935
12936    let open = state.open_findings();
12937    if !open.is_empty() {
12938        body.push_str("## Open review findings\n\n");
12939        for f in &open {
12940            body.push_str(&format!(
12941                "- `{}` [{:?}] {}\n",
12942                f.id, f.severity, "Context retained in local review records."
12943            ));
12944        }
12945        body.push('\n');
12946    }
12947
12948    if let Some(fix) = fix
12949        && !fix.rejected.is_empty()
12950    {
12951        body.push_str("## Declined by the fixer\n\n");
12952        for r in &fix.rejected {
12953            body.push_str(&format!(
12954                "- `{}`: {}\n",
12955                r.id, "Reason retained in local review records."
12956            ));
12957        }
12958        body.push('\n');
12959    }
12960
12961    if review {
12962        // The review prompt is not the task; list what the branch carries.
12963        body.push_str("## Commits under review\n\n");
12964        if let Some(facts) = facts {
12965            let mut left = COMMITS_MAX_CHARS;
12966            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12967                let mut entry = format!("- {}\n", markdown_safe(subject));
12968                for l in markdown_safe(text).lines() {
12969                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12970                }
12971                if left == 0 {
12972                    body.push_str(&format!(
12973                        "- ... {} more commit(s)\n",
12974                        facts.commits.len() - i
12975                    ));
12976                    break;
12977                }
12978                if entry.len() > left {
12979                    // Even the first commit is cut: one huge body must not
12980                    // push the whole description past GitHub's limit.
12981                    let mut end = left;
12982                    while !entry.is_char_boundary(end) {
12983                        end -= 1;
12984                    }
12985                    entry.truncate(end);
12986                    entry.push_str("\n  ... (truncated)\n");
12987                    left = 0;
12988                } else {
12989                    left -= entry.len();
12990                }
12991                body.push_str(&entry);
12992            }
12993            if !facts.stat.trim().is_empty() {
12994                body.push_str(&format!(
12995                    "\n## Diff stat\n\n```\n{}\n```\n",
12996                    markdown_safe(facts.stat.trim())
12997                ));
12998            }
12999        } else {
13000            match &state.reviewed_commits {
13001                Some(subjects) => {
13002                    for s in subjects {
13003                        body.push_str(&format!("- {}\n", s.trim()));
13004                    }
13005                }
13006                None => {
13007                    // An older run kept only the prompt, with the commit list
13008                    // after its first paragraph.
13009                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13010                    body.push_str(rest.trim());
13011                    body.push('\n');
13012                }
13013            }
13014        }
13015    } else {
13016        let task = state.instruction.trim();
13017        let task = if task.is_empty() {
13018            "(empty task)"
13019        } else {
13020            task
13021        };
13022        body.push_str(&format!(
13023            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13024            task.replace("</details>", "&lt;/details&gt;")
13025        ));
13026    }
13027
13028    body.push_str(&format!(
13029        "\n---\nmagi:run/{} magi:candidate-{}\n",
13030        state.id,
13031        winner.to_ascii_lowercase()
13032    ));
13033
13034    PrMessage { title, body }
13035}
13036
13037/// Source imports have already been prepared from structured inputs. Legacy
13038/// inferred references must never instruct a seat to reconstruct those imports.
13039fn seeded_instruction(state: &RunState) -> String {
13040    let mut inputs = Vec::new();
13041    if let Some(start) = &state.sources.start_from {
13042        inputs.push(format!(
13043            "Start from `{}` at `{}` (including ancestry).",
13044            start.token, start.sha
13045        ));
13046    }
13047    for pick in &state.sources.pick_commits {
13048        inputs.push(format!(
13049            "Pick `{}` at `{}` (single commit).",
13050            pick.token, pick.sha
13051        ));
13052    }
13053    if inputs.is_empty() {
13054        state.instruction.clone()
13055    } else {
13056        format!(
13057            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13058            state.instruction,
13059            inputs.join("\n")
13060        )
13061    }
13062}
13063
13064/// Does the winner have no commits ahead of the base it would land on?
13065/// Any failure to find out reads as "not empty": the merge then behaves as it
13066/// always did rather than refusing on a guess.
13067async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13068    let base = &state.base_branch;
13069    let mut against = base.clone();
13070    if mode == MergeMode::Pr {
13071        // A pull request lands on the remote's base, never the local branch
13072        // of the same name: if that cannot be read, "not empty" is the safe
13073        // answer.
13074        let remote = &state.config.merge.remote;
13075        let tracking = format!("{remote}/{base}");
13076        let fetched = git::fetch(repo, remote, base).await;
13077        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13078            against = tracking;
13079        } else {
13080            return false;
13081        }
13082    }
13083    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13084}
13085
13086/// Why nothing was opened for an empty winner, with what the task's own
13087/// references resolved to.
13088fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13089    let mut detail = format!(
13090        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13091         nothing to open a pull request for"
13092    );
13093    detail.push_str(
13094        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13095    );
13096    detail
13097}
13098
13099/// What the `Pr` merge does once it knows whether the branch already has an
13100/// open pull request.
13101#[derive(Debug, PartialEq, Eq)]
13102enum PrPlan {
13103    Create,
13104    Adopt { url: String, title: String },
13105    Stop(String),
13106}
13107
13108/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13109/// a failed lookup -> stop with the real reason. Never guesses.
13110fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13111    match found {
13112        Ok(land::OpenPr::None) => PrPlan::Create,
13113        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13114        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13115            "several open pull requests exist for this branch, not picking one: {}",
13116            urls.join(" ")
13117        )),
13118        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13119    }
13120}
13121
13122/// `gh pr create`, returning the PR url.
13123pub(crate) async fn gh_pr_create(
13124    cwd: &Path,
13125    base: &str,
13126    head: &str,
13127    title: &str,
13128    body: &str,
13129) -> Result<String> {
13130    let out = tokio::process::Command::new("gh")
13131        .args([
13132            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13133        ])
13134        .current_dir(cwd)
13135        .quiet()
13136        .stdin(std::process::Stdio::null())
13137        .output()
13138        .await
13139        .context("spawn gh")?;
13140    if out.status.success() {
13141        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13142    } else {
13143        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13144    }
13145}
13146
13147/// Never force deletion of uncommitted work, including unknown legacy trees.
13148async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13149    match std::fs::symlink_metadata(path) {
13150        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13151            // A previous cleanup may have removed the tree before a branch
13152            // deletion or state save failed. Unregister it if still present.
13153            git::worktree_remove_clean(repo, path).await?;
13154            return Ok(());
13155        }
13156        Err(e) => return Err(e.into()),
13157        Ok(_) => {}
13158    }
13159    if !git::is_clean(path).await? {
13160        bail!(
13161            "worktree has uncommitted work; refusing removal: {}",
13162            path.display()
13163        );
13164    }
13165    if !git::worktree_remove_clean(repo, path).await? {
13166        bail!("worktree removal refused: {}", path.display());
13167    }
13168    Ok(())
13169}
13170
13171/// Tear a run's worktrees and branches down.
13172///
13173/// `home` is where the updated `run.json` is saved (via
13174/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13175/// a housekeeping pass already has its own honest `home` handed to it, and
13176/// falling through to the global here would write back through whichever
13177/// directory some other process or test pinned into that `OnceLock` first,
13178/// not the one the caller actually resolved its `runs` and `state` from.
13179pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13180    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13181        bail!("refusing fold: {reason}");
13182    }
13183    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13184        bail!("candidate preservation incomplete; refusing cleanup");
13185    }
13186    // Serialize ordinary folding with the same starts/resumes as target-only
13187    // eviction. Otherwise a second janitor could remove the whole worktree
13188    // around a live output borrower even though target eviction was fenced.
13189    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13190    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13191    let _driver = crate::driver::maintenance(home, &state.id)?;
13192    let mut output_guards = Vec::new();
13193    for entry in std::fs::read_dir(state.worktree_root())
13194        .into_iter()
13195        .flatten()
13196        .flatten()
13197    {
13198        if entry.path().join(".git").is_file() {
13199            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13200        }
13201    }
13202    let repo = state.repo.clone();
13203    // CLI, Web and automatic cleanup all enter here, independently of
13204    // Runner::fold_losers. Protect deletion and shared config release too.
13205    let wait = repository_wait(state, None);
13206    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13207        Ok(lease) => lease,
13208        Err(e) => {
13209            state.save_under(home)?;
13210            return Err(e);
13211        }
13212    };
13213    // Confirmed merges dispose of their owned sources after durable evidence
13214    // capture. Preflight every source before removing anything.
13215    let _source_outputs = if state.status == RunStatus::Merged
13216        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13217    {
13218        verify_merged_fold_sources(state, drop_winner, home).await?
13219    } else {
13220        Vec::new()
13221    };
13222    let root = state.worktree_root();
13223    let winner = state.tally.as_ref().map(|t| t.winner);
13224    let mut removed = Vec::new();
13225
13226    for i in 0..state.candidates.len() {
13227        let c = state.candidates[i].clone();
13228        let is_winner = Some(c.label) == winner;
13229        if is_winner && !drop_winner {
13230            continue;
13231        }
13232        let existed = c.worktree.exists();
13233        // A vanished directory can still have a Git registration that keeps
13234        // its branch checked out. Remove that registration before the branch.
13235        remove_preserved_worktree(&repo, &c.worktree).await?;
13236        if existed {
13237            removed.push(c.worktree.to_string_lossy().into_owned());
13238        }
13239        // A branch handed to a later run (and its pull request) is not this
13240        // run's to delete.
13241        let handed_over = state.released_branches.contains(&c.branch);
13242        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13243            if !git::branch_delete(&repo, &c.branch).await? {
13244                bail!("candidate branch removal refused");
13245            }
13246            removed.push(c.branch.clone());
13247        }
13248        state.candidates[i].folded = true;
13249    }
13250
13251    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13252        let path = name.path();
13253        let keep = !drop_winner
13254            && winner.is_some_and(|w| {
13255                path.file_name()
13256                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13257            });
13258        if keep {
13259            continue;
13260        }
13261        remove_preserved_worktree(&repo, &path).await?;
13262        removed.push(path.to_string_lossy().into_owned());
13263    }
13264
13265    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13266    // judge worktrees, so once the loop above has cleared all of them out,
13267    // the parent is a bare directory nobody else was ever going to remove -
13268    // git only ever managed what was inside it. Left alone, one of these
13269    // accumulates per fully-folded run; the operator's own machine had 74.
13270    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13271    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13272    // so does anything a run never claimed that happens to share the bay.
13273    remove_if_empty(&root);
13274
13275    if state.enabled_worktree_config && drop_winner {
13276        // A release, not a raw disable: some sibling run in this repository
13277        // may still hold its own reference (see `git::acquire_worktree_config`),
13278        // and only the last release actually turns the setting back off.
13279        git::release_worktree_config(&repo).await.ok();
13280        state.enabled_worktree_config = false;
13281    }
13282    state.save_under(home)?;
13283    Ok(removed)
13284}
13285
13286async fn verify_merged_fold_sources(
13287    state: &RunState,
13288    drop_winner: bool,
13289    home: &Path,
13290) -> Result<Vec<std::fs::File>> {
13291    let winner = state.tally.as_ref().map(|t| t.winner);
13292    let mut heads = BTreeSet::new();
13293    for candidate in &state.candidates {
13294        if Some(candidate.label) == winner && !drop_winner {
13295            continue;
13296        }
13297        anyhow::ensure!(
13298            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13299            "candidate recovery source is outside the inventoried run root: {}",
13300            candidate.worktree.display()
13301        );
13302        match std::fs::symlink_metadata(&candidate.worktree) {
13303            Ok(_) => {}
13304            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13305            Err(error) => return Err(error.into()),
13306        }
13307        if !state.released_branches.contains(&candidate.branch)
13308            && git::branch_exists(&state.repo, &candidate.branch).await?
13309        {
13310            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13311        }
13312    }
13313    let mut cargo_guards = Vec::new();
13314    let entries = match std::fs::read_dir(state.worktree_root()) {
13315        Ok(entries) => Some(entries),
13316        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13317        Err(error) => return Err(error.into()),
13318    };
13319    for entry in entries.into_iter().flatten() {
13320        let path = entry?.path();
13321        let metadata = std::fs::symlink_metadata(&path)?;
13322        anyhow::ensure!(
13323            metadata.is_dir() && !metadata.file_type().is_symlink(),
13324            "non-directory or symlink recovery source retained: {}",
13325            path.display()
13326        );
13327        if !drop_winner
13328            && winner.is_some_and(|w| {
13329                path.file_name()
13330                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13331            })
13332        {
13333            continue;
13334        }
13335        anyhow::ensure!(
13336            path.join(".git").is_file(),
13337            "unregistered recovery source retained: {}",
13338            path.display()
13339        );
13340        anyhow::ensure!(
13341            git::git(
13342                &path,
13343                &[
13344                    "status",
13345                    "--porcelain",
13346                    "--untracked-files=all",
13347                    "--ignored",
13348                    "--",
13349                    ".",
13350                    ":(exclude)target"
13351                ]
13352            )
13353            .await?
13354            .is_empty(),
13355            "source changes or ignored recovery files retained: {}",
13356            path.display()
13357        );
13358        match std::fs::symlink_metadata(path.join("target")) {
13359            Ok(_) => {
13360                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13361            }
13362            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13363            Err(error) => return Err(error.into()),
13364        }
13365        for checkpoint in [
13366            "rebase-merge",
13367            "rebase-apply",
13368            "sequencer",
13369            "CHERRY_PICK_HEAD",
13370            "MERGE_HEAD",
13371            "REBASE_HEAD",
13372            "REVERT_HEAD",
13373            "BISECT_LOG",
13374            "AUTO_MERGE",
13375            "MERGE_MSG",
13376        ] {
13377            let location = git::git(
13378                &path,
13379                &[
13380                    "rev-parse",
13381                    "--path-format=absolute",
13382                    "--git-path",
13383                    checkpoint,
13384                ],
13385            )
13386            .await?;
13387            anyhow::ensure!(
13388                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13389                if e.kind() == std::io::ErrorKind::NotFound),
13390                "Git recovery checkpoint retained: {checkpoint}"
13391            );
13392        }
13393        heads.insert(git::rev_parse(&path, "HEAD").await?);
13394    }
13395    crate::fold_evidence::preserve(state, home, &heads).await?;
13396
13397    Ok(cargo_guards)
13398}
13399
13400/// Remove `dir` if it exists and has nothing in it.
13401///
13402/// Best-effort and silent by design: a directory that is not empty (a run
13403/// whose winner is still parked there, a stray file some other process left)
13404/// is exactly the case this must refuse, and a directory that is already gone
13405/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13406/// already refuses a non-empty directory, so the emptiness check below is
13407/// belt, not suspenders - it is what keeps this from ever attempting the
13408/// removal in the case that matters, rather than trusting `remove_dir`'s
13409/// error path to have no side effects if it ever changed.
13410fn remove_if_empty(dir: &Path) {
13411    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13412        std::fs::remove_dir(dir).ok();
13413    }
13414}
13415
13416/// Severity of the worst open finding in the last review round, for reporting.
13417pub fn worst_open(state: &RunState) -> Option<Severity> {
13418    state
13419        .reviews
13420        .last()?
13421        .reviews
13422        .iter()
13423        .flat_map(|r| r.findings.iter())
13424        .map(|f| f.severity)
13425        .max()
13426}
13427
13428/// `[worktree] setup` hides its products per worktree, which needs
13429/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13430/// worktrees, like the hook's; every place that runs setup calls this first.
13431async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13432    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13433        git::acquire_worktree_config(repo).await?;
13434        state.enabled_worktree_config = true;
13435    }
13436    Ok(())
13437}
13438
13439#[cfg(test)]
13440mod tests {
13441    #[tokio::test]
13442    async fn launcher_base_fetch_requires_remote_success() {
13443        let tmp = tempfile::tempdir().unwrap();
13444        let repo = tmp.path().join("repo");
13445        let remote = tmp.path().join("remote");
13446        std::fs::create_dir(&repo).unwrap();
13447        std::fs::create_dir(&remote).unwrap();
13448        git::git(&remote, &["init", "--bare", "-b", "main"])
13449            .await
13450            .unwrap();
13451        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13452        git::git(
13453            &repo,
13454            &[
13455                "-c",
13456                "user.name=Fixture",
13457                "-c",
13458                "user.email=fixture@example.invalid",
13459                "commit",
13460                "--allow-empty",
13461                "-m",
13462                "fixture",
13463            ],
13464        )
13465        .await
13466        .unwrap();
13467        git::git(
13468            &repo,
13469            &["remote", "add", "origin", remote.to_str().unwrap()],
13470        )
13471        .await
13472        .unwrap();
13473        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13474        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13475        assert_eq!(
13476            resolve_base(&repo, "main", "origin", None)
13477                .await
13478                .unwrap()
13479                .commit,
13480            head
13481        );
13482        #[cfg(unix)]
13483        {
13484            let policy: crate::config::Coordination = toml::from_str(
13485                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13486            ).unwrap();
13487            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13488            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13489            let held = crate::repo_lock::SeatLease::operation_with(
13490                &repo,
13491                Some(&policy),
13492                &crate::repo_lock::WaitPolicy::default(),
13493            )
13494            .await
13495            .unwrap();
13496            assert!(held.is_some());
13497            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13498            tokio::pin!(fetch);
13499            assert!(
13500                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13501                    .await
13502                    .is_err()
13503            );
13504            assert!(
13505                !repo.join(".git/FETCH_HEAD").exists(),
13506                "no fetch before admission"
13507            );
13508            drop(held);
13509            let fetched = fetch.await.unwrap();
13510            assert_eq!(fetched.commit, head);
13511            assert_eq!(fetched.remote, "origin");
13512            assert_eq!(fetched.branch, "main");
13513        }
13514        // A surviving tracking ref is not successful fetch evidence.
13515        git::git(
13516            &repo,
13517            &[
13518                "remote",
13519                "set-url",
13520                "origin",
13521                tmp.path().join("missing").to_str().unwrap(),
13522            ],
13523        )
13524        .await
13525        .unwrap();
13526        let error = resolve_base(&repo, "main", "origin", None)
13527            .await
13528            .unwrap_err()
13529            .to_string();
13530        assert!(error.contains("cannot read origin/main"));
13531        assert!(error.contains("refusing to branch off"));
13532    }
13533
13534    #[test]
13535    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13536        let leaked = vec!["chore(deps): update a crate".to_owned()];
13537        let own = "fix(daemon): apply a chosen action";
13538        assert!(should_retitle("", own, &leaked));
13539        assert!(should_retitle(
13540            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13541            own,
13542            &leaked
13543        ));
13544        assert!(should_retitle(
13545            "chore: land candidate A of run 1",
13546            own,
13547            &leaked
13548        ));
13549        assert!(should_retitle(
13550            "magi: candidate A (uncommitted work)",
13551            own,
13552            &leaked
13553        ));
13554        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13555        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13556        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13557    }
13558
13559    #[test]
13560    fn pr_merge_plan_creates_adopts_or_stops() {
13561        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13562        assert_eq!(
13563            pr_merge_plan(Ok(land::OpenPr::One {
13564                url: "u".into(),
13565                title: "t".into()
13566            })),
13567            PrPlan::Adopt {
13568                url: "u".into(),
13569                title: "t".into()
13570            }
13571        );
13572        let PrPlan::Stop(many) =
13573            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13574        else {
13575            panic!("many must stop");
13576        };
13577        assert!(many.contains('a') && many.contains('b'));
13578        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13579            panic!("a failed lookup must stop");
13580        };
13581        assert!(err.contains("bad token"));
13582    }
13583
13584    use super::*;
13585    use crate::run::GateStatus;
13586    use std::collections::BTreeMap;
13587    use std::time::Duration;
13588
13589    fn conductor() -> AgentSpec {
13590        AgentSpec {
13591            id: "conductor".to_owned(),
13592            kind: crate::config::AgentKind::Command,
13593            model: None,
13594            command: vec!["true".to_owned()],
13595            extra_args: Vec::new(),
13596            env: BTreeMap::new(),
13597            prompt_delivery: None,
13598            max_turn_secs: None,
13599        }
13600    }
13601
13602    fn spec(id: &str) -> AgentSpec {
13603        AgentSpec {
13604            id: id.to_owned(),
13605            kind: crate::config::AgentKind::Command,
13606            model: None,
13607            command: vec!["true".to_owned()],
13608            extra_args: Vec::new(),
13609            env: BTreeMap::new(),
13610            prompt_delivery: None,
13611            max_turn_secs: None,
13612        }
13613    }
13614
13615    fn ids(xs: &[&str]) -> BTreeSet<String> {
13616        xs.iter().map(|s| (*s).to_owned()).collect()
13617    }
13618
13619    #[test]
13620    fn pick_successor_skips_an_agent_another_seat_holds() {
13621        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13622        let roster = [spec("a"), spec("b"), spec("c")];
13623        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13624        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13625    }
13626
13627    #[test]
13628    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13629        // The other seat started on c but was handed to d; c is free again.
13630        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13631        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13632        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13633        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13634        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13635    }
13636
13637    #[test]
13638    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13639        let roster = [spec("a"), spec("b")];
13640        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13641        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13642        let carried = ids(&["b"]);
13643        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13644        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13645    }
13646
13647    #[test]
13648    fn pick_successor_returns_none_without_any_untried_successor() {
13649        let roster = [spec("a"), spec("b")];
13650        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13651        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13652        let carried = ids(&["a"]);
13653        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13654    }
13655
13656    #[test]
13657    fn pick_successor_rescue_avoids_another_seats_occupant() {
13658        // b is a carried failure and free; a is held by the other seat.
13659        let roster = [spec("a"), spec("b"), spec("c")];
13660        let carried = ids(&["b", "c"]);
13661        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13662        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13663    }
13664
13665    #[test]
13666    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13667        let roster = [spec("a"), spec("b"), spec("c")];
13668        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13669        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13670    }
13671
13672    #[test]
13673    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13674        let roster = [spec("a"), spec("b"), spec("c")];
13675        let failed = ids(&["a", "b", "c"]);
13676        // Rescue looks at the whole roster, once per id, then runs out.
13677        let mut tried = ids(&["b"]);
13678        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13679        assert_eq!(first.id, "a");
13680        tried.insert(first.id.clone());
13681        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13682        assert_eq!(second.id, "c");
13683        tried.insert(second.id.clone());
13684        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13685    }
13686
13687    #[test]
13688    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13689        let roster = [spec("a"), spec("b")];
13690        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13691        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13692    }
13693
13694    #[test]
13695    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13696        let roster = [spec("a"), spec("b"), spec("c")];
13697        let h = SeatHistory {
13698            failed: ids(&["a"]),
13699            last_ok: Some("b".to_owned()),
13700            last_fail: None,
13701        };
13702        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13703        // A last answerer that left the roster, or later failed, is ignored.
13704        let gone = SeatHistory {
13705            last_ok: Some("zzz".to_owned()),
13706            ..h.clone()
13707        };
13708        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13709        let failed = SeatHistory {
13710            failed: ids(&["a", "b"]),
13711            last_ok: Some("b".to_owned()),
13712            last_fail: None,
13713        };
13714        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13715    }
13716
13717    #[test]
13718    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13719        let first = SeatState::new("review-1", "alpha", 7);
13720        let next = handover_seat("review-1", "gamma", 7);
13721        assert_ne!(first.claude_session, next.claude_session);
13722    }
13723
13724    #[test]
13725    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13726        let mut state = state_with_summary("x", "y");
13727        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13728        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13729        assert_ne!(a.claude_session, b.claude_session);
13730    }
13731
13732    #[test]
13733    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13734        let roster = [spec("a"), spec("b")];
13735        let h = SeatHistory {
13736            failed: ids(&["a", "b"]),
13737            ..SeatHistory::default()
13738        };
13739        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13740        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13741        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13742    }
13743
13744    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13745    // fallback loop depends on to terminate: it must walk forward from the
13746    // seat's own position, never restart at the front of the roster, and it
13747    // must never hand back an id already tried, however many times that id
13748    // happens to appear.
13749
13750    #[test]
13751    fn failure_signature_ignores_numbers_and_paths() {
13752        assert_eq!(
13753            failure_signature("exited with Some(2) and no usable output"),
13754            failure_signature("exited with Some(137) and no usable output")
13755        );
13756        assert_eq!(
13757            failure_signature("process exited with no usable output (exit code 2)"),
13758            failure_signature("process exited with no usable output (exit code 137)")
13759        );
13760        for (old, new) in [
13761            (
13762                "exited with Some(2) and no usable output",
13763                "process exited with no usable output (exit code 2)",
13764            ),
13765            (
13766                "exited with None and no usable output",
13767                "process exited with no usable output (exit code unavailable)",
13768            ),
13769        ] {
13770            let previous = FailClass::Other(failure_signature(old));
13771            let current = FailClass::Other(failure_signature(new));
13772            assert_eq!(previous, current, "resumption retains the failure class");
13773            assert!(!should_hand_over(Some(&previous), &current));
13774        }
13775        assert_eq!(
13776            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13777            failure_signature("cannot open /var/x.txt: denied")
13778        );
13779        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13780    }
13781
13782    #[test]
13783    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13784        let other = FailClass::Other("x".into());
13785        assert!(should_hand_over(None, &FailClass::Quota));
13786        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13787        assert!(should_hand_over(
13788            Some(&FailClass::Timeout),
13789            &FailClass::Timeout
13790        ));
13791        assert!(should_hand_over(None, &other));
13792        assert!(!should_hand_over(Some(&other), &other));
13793        assert!(should_hand_over(
13794            Some(&other),
13795            &FailClass::Other("y".into())
13796        ));
13797        // A quota or timeout in between ends the run of identical failures.
13798        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13799        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13800    }
13801
13802    #[test]
13803    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13804        let a = SeatState::new("judge-1", "alpha", 7);
13805        let b = handover_seat("judge-1", "beta", 7);
13806        assert_ne!(a.claude_session, b.claude_session);
13807        assert_eq!(b.turns, 0);
13808    }
13809
13810    #[test]
13811    fn a_timeout_is_classified_apart_from_other_failures() {
13812        assert_eq!(
13813            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13814            Some(FailClass::Timeout)
13815        );
13816        assert!(matches!(
13817            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13818            Some(FailClass::Other(_))
13819        ));
13820    }
13821
13822    #[test]
13823    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13824        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13825        let tried = BTreeSet::from(["beta".to_owned()]);
13826        // beta sits at index 1; the next candidate is gamma, never alpha —
13827        // which is very likely a different candidate slot's own agent.
13828        let next = next_untried_in_roster(&roster, 1, &tried);
13829        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13830    }
13831
13832    #[test]
13833    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13834        let roster = vec![spec("alpha"), spec("beta")];
13835        let tried = BTreeSet::from(["beta".to_owned()]);
13836        // beta is the roster's last entry: nothing follows it, and alpha —
13837        // earlier in the roster, almost certainly a different candidate
13838        // slot's own agent — must not be reached by wrapping back to it.
13839        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13840    }
13841
13842    #[test]
13843    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13844        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13845        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13846        // beta (index 1) and gamma (index 2, the only entry after it) have
13847        // both been tried; alpha (index 0) never has, but it comes before
13848        // beta's own position, so there is nothing further for this seat.
13849        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13850    }
13851
13852    #[test]
13853    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13854        let roster = vec![spec("a"), spec("a"), spec("b")];
13855        let tried = BTreeSet::from(["a".to_owned()]);
13856        let next = next_untried_in_roster(&roster, 0, &tried);
13857        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13858    }
13859
13860    #[test]
13861    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13862        let roster = vec![spec("a"), spec("b")];
13863        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13864        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13865    }
13866
13867    #[test]
13868    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13869        let dir = tempfile::tempdir().unwrap();
13870        let bay = dir.path().join("ffff");
13871
13872        // Not there yet: nothing to do, nothing to panic on.
13873        remove_if_empty(&bay);
13874        assert!(!bay.exists());
13875
13876        // Something still inside - the winner's worktree, or a stray file -
13877        // keeps the directory standing.
13878        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13879        remove_if_empty(&bay);
13880        assert!(bay.exists(), "non-empty directory must survive");
13881
13882        // Once the last entry is gone, so is the directory itself.
13883        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13884        remove_if_empty(&bay);
13885        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13886    }
13887
13888    // `round_is_clean` is the exact decision this task fixed: a round with a
13889    // seat that never answered must not read the same as a round every seat
13890    // actually reviewed. These are deterministic and process-free by design —
13891    // the equivalent end-to-end check (a real reviewer timing out under a
13892    // live graph run) is a genuine race against wall-clock contention, and a
13893    // spawn slow enough to blow even a generous budget under a loaded test
13894    // run must not turn this specific regression check flaky.
13895
13896    #[test]
13897    fn a_full_panel_that_found_nothing_is_clean() {
13898        assert!(round_is_clean(
13899            0,
13900            true,
13901            2,
13902            2,
13903            0,
13904            IncompleteReviewPolicy::Block
13905        ));
13906    }
13907
13908    #[test]
13909    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13910        assert!(!round_is_clean(
13911            0,
13912            true,
13913            1,
13914            2,
13915            0,
13916            IncompleteReviewPolicy::Block
13917        ));
13918    }
13919
13920    #[test]
13921    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13922        assert!(!round_is_clean(
13923            1,
13924            true,
13925            1,
13926            2,
13927            0,
13928            IncompleteReviewPolicy::Warn
13929        ));
13930    }
13931
13932    #[test]
13933    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13934        assert!(round_is_clean(
13935            0,
13936            true,
13937            1,
13938            2,
13939            0,
13940            IncompleteReviewPolicy::Warn
13941        ));
13942    }
13943
13944    #[test]
13945    fn a_full_panel_with_an_open_finding_is_not_clean() {
13946        assert!(!round_is_clean(
13947            1,
13948            true,
13949            2,
13950            2,
13951            0,
13952            IncompleteReviewPolicy::Block
13953        ));
13954    }
13955
13956    #[test]
13957    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13958        assert!(!round_is_clean(
13959            0,
13960            false,
13961            2,
13962            2,
13963            0,
13964            IncompleteReviewPolicy::Block
13965        ));
13966    }
13967
13968    // The stall this task closes: under the default `block` policy, a seat
13969    // missing only because it was rate limited must not force a wait for a
13970    // session limit that will not lift by the next round. `round_is_clean`
13971    // is where that quorum carve-out lives; the review loop around it never
13972    // changes what a reviewer's vote or a finding's severity means.
13973
13974    #[test]
13975    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13976        // 1 of 2 answered, and the one missing was quota'd — the exact
13977        // "review-2 rate limited (quota)" shape from the field report.
13978        assert!(round_is_clean(
13979            0,
13980            true,
13981            1,
13982            2,
13983            1,
13984            IncompleteReviewPolicy::Block
13985        ));
13986    }
13987
13988    #[test]
13989    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13990        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13991        // not a quota loss (`quota_missing` stays 0) — worth another try.
13992        assert!(!round_is_clean(
13993            0,
13994            true,
13995            1,
13996            2,
13997            0,
13998            IncompleteReviewPolicy::Block
13999        ));
14000    }
14001
14002    #[test]
14003    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14004        assert!(!round_is_clean(
14005            1,
14006            true,
14007            1,
14008            2,
14009            1,
14010            IncompleteReviewPolicy::Block
14011        ));
14012        assert!(!round_is_clean(
14013            0,
14014            false,
14015            1,
14016            2,
14017            1,
14018            IncompleteReviewPolicy::Block
14019        ));
14020    }
14021
14022    #[test]
14023    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14024        // Every seat quota'd, nobody answered: there is no panel to decide
14025        // on, so this must fall through to the existing block-and-retry
14026        // fallback rather than call an unreviewed patch clean.
14027        assert!(!round_is_clean(
14028            0,
14029            true,
14030            0,
14031            2,
14032            2,
14033            IncompleteReviewPolicy::Block
14034        ));
14035    }
14036
14037    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14038        CommandOutcome {
14039            command: "test".to_owned(),
14040            code,
14041            output_tail: String::new(),
14042            duration_ms: 0,
14043            resource_blocked,
14044        }
14045    }
14046
14047    #[cfg(unix)]
14048    #[tokio::test]
14049    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14050        ask_test_home();
14051        let temp = tempfile::tempdir().unwrap();
14052        let mut runner = runner_at(RunStatus::Gating);
14053        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14054        let commands = vec![
14055            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14056            "touch must-not-run".to_owned(),
14057        ];
14058        let (outcomes, retried, pids) = run_e2e_with_retry(
14059            &mut runner.state,
14060            &shell,
14061            &commands,
14062            temp.path(),
14063            Duration::from_secs(5),
14064            "fixture",
14065        )
14066        .await;
14067        assert_eq!(outcomes.len(), 1);
14068        assert!(outcomes[0].resource_blocked);
14069        assert!(!outcomes[0].build_failed());
14070        assert!(!retried);
14071        assert!(pids.is_empty());
14072        assert!(!temp.path().join("must-not-run").exists());
14073        assert_eq!(
14074            runner.state.resource_block.as_ref().unwrap().kind,
14075            "incremental cache admission"
14076        );
14077    }
14078
14079    #[test]
14080    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14081        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14082        assert!(
14083            !verify_inconclusive(&[outcome(Some(1), false)]),
14084            "an ordinary failure is still evidence about the patch"
14085        );
14086        assert!(verify_inconclusive(&[outcome(None, true)]));
14087        assert!(
14088            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14089            "one inconclusive outcome taints the whole batch"
14090        );
14091        assert!(!verify_inconclusive(&[]));
14092    }
14093
14094    #[tokio::test]
14095    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14096        // Alive for the first two checks, then dead - confirms the loop
14097        // actually re-polls rather than deciding once and sleeping out the
14098        // ceiling regardless.
14099        let calls = std::sync::atomic::AtomicUsize::new(0);
14100        let started = Instant::now();
14101        wait_for_pids_with(
14102            &[123],
14103            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14104            Duration::from_millis(5),
14105            Duration::from_secs(5),
14106        )
14107        .await;
14108        assert!(
14109            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14110            "must keep checking rather than deciding on the first answer"
14111        );
14112        assert!(
14113            started.elapsed() < Duration::from_secs(1),
14114            "must return the moment it is confirmed dead, not wait out the ceiling"
14115        );
14116    }
14117
14118    #[tokio::test]
14119    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14120        let started = Instant::now();
14121        wait_for_pids_with(
14122            &[123],
14123            |_| true, // never reports dead
14124            Duration::from_millis(5),
14125            Duration::from_millis(30),
14126        )
14127        .await;
14128        let elapsed = started.elapsed();
14129        assert!(
14130            elapsed >= Duration::from_millis(30),
14131            "must not give up before its own ceiling: {elapsed:?}"
14132        );
14133        assert!(
14134            elapsed < Duration::from_secs(1),
14135            "must not wait past its own ceiling either: {elapsed:?}"
14136        );
14137    }
14138
14139    #[tokio::test]
14140    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14141        let started = Instant::now();
14142        wait_for_pids_with(
14143            &[],
14144            |_| true,
14145            Duration::from_secs(5),
14146            Duration::from_secs(5),
14147        )
14148        .await;
14149        assert!(
14150            started.elapsed() < Duration::from_millis(200),
14151            "an empty pid list has nothing to confirm"
14152        );
14153    }
14154
14155    // `review_conclusion` is the exact decision the review hand-off task
14156    // fixed: a round budget spent (or a tree that stopped moving) must not
14157    // collapse into `Blocked` regardless of what verification actually
14158    // said. Deterministic and process-free for the same reason the
14159    // `round_is_clean` family above is.
14160    fn review_round(
14161        clean: bool,
14162        blocking: usize,
14163        answered: usize,
14164        expected: usize,
14165        progressed: bool,
14166        e2e_ok: bool,
14167    ) -> ReviewRound {
14168        ReviewRound {
14169            round: 1,
14170            head: "h".to_owned(),
14171            verified_head: None,
14172            verified_at: None,
14173            reviews: Vec::new(),
14174            e2e: vec![CommandOutcome {
14175                command: "test".to_owned(),
14176                code: Some(if e2e_ok { 0 } else { 1 }),
14177                output_tail: String::new(),
14178                duration_ms: 0,
14179                resource_blocked: false,
14180            }],
14181            verify_retried: false,
14182            e2e_deferred: false,
14183            e2e_defer_reason: None,
14184            fix: None,
14185            blocking,
14186            answered,
14187            expected,
14188            clean,
14189            progressed,
14190            vote_split: false,
14191            reconsideration: Vec::new(),
14192            verdict: None,
14193        }
14194    }
14195
14196    #[test]
14197    fn review_conclusion_is_none_when_nothing_has_run() {
14198        assert_eq!(review_conclusion(&[], 3), None);
14199    }
14200
14201    #[test]
14202    fn review_conclusion_is_none_while_rounds_remain() {
14203        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14204        assert_eq!(review_conclusion(&rounds, 3), None);
14205    }
14206
14207    #[test]
14208    fn review_conclusion_is_gating_once_a_round_is_clean() {
14209        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14210        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14211    }
14212
14213    #[test]
14214    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14215        let rounds = vec![
14216            review_round(false, 1, 2, 2, true, true),
14217            review_round(false, 1, 2, 2, true, true),
14218        ];
14219        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14220    }
14221
14222    #[test]
14223    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14224        let rounds = vec![
14225            review_round(false, 1, 2, 2, true, true),
14226            review_round(false, 1, 2, 2, true, false),
14227        ];
14228        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14229    }
14230
14231    #[test]
14232    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14233        // Magi never got a command to run against this round's own head — a
14234        // resource-blocked attempt, not a red one — so this must never
14235        // settle on `Blocked` the way a genuine e2e failure would. `None`
14236        // here is what tells `Runner::review_loop` to retry the check
14237        // itself rather than trust this cheap recomputation with a verdict
14238        // it cannot actually produce.
14239        let mut blocked = review_round(false, 1, 2, 2, true, false);
14240        blocked.e2e[0].resource_blocked = true;
14241        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14242        assert_eq!(review_conclusion(&rounds, 2), None);
14243    }
14244
14245    #[test]
14246    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14247        // Missing input, not a verified tree — never a hand-off candidate.
14248        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14249        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14250    }
14251
14252    #[test]
14253    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14254        let rounds = vec![
14255            review_round(false, 1, 2, 2, false, true),
14256            review_round(false, 1, 2, 2, false, true),
14257        ];
14258        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14259    }
14260
14261    fn secs(n: u64) -> Duration {
14262        Duration::from_secs(n)
14263    }
14264
14265    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14266    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14267    #[tokio::test]
14268    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14269        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14270        if std::env::var_os(CHILD).is_none() {
14271            // Other tests reconcile every run in the process-global home,
14272            // briefly taking owner locks. This ancestry test expects an idle
14273            // predecessor, so give it its own process and pinned test home.
14274            let output = std::process::Command::new(std::env::current_exe().unwrap())
14275                .args([
14276                    "--exact",
14277                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14278                    "--nocapture",
14279                ])
14280                .env(CHILD, "1")
14281                .env_remove("MAGI_WORKER_HOME")
14282                .env_remove("MAGI_WORKER_ID")
14283                .output()
14284                .unwrap();
14285            assert!(
14286                output.status.success()
14287                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14288                "{}\n{}",
14289                String::from_utf8_lossy(&output.stdout),
14290                String::from_utf8_lossy(&output.stderr)
14291            );
14292            return;
14293        }
14294        ask_test_home();
14295        let tmp = tempfile::tempdir().unwrap();
14296        let repo = tmp.path().join("repo");
14297        std::fs::create_dir_all(&repo).unwrap();
14298        init_repo(&repo);
14299        let g = |args: &[&str]| {
14300            let out = std::process::Command::new("git")
14301                .args(args)
14302                .current_dir(&repo)
14303                .quiet()
14304                .output()
14305                .unwrap();
14306            assert!(
14307                out.status.success(),
14308                "{}",
14309                String::from_utf8_lossy(&out.stderr)
14310            );
14311        };
14312        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14313        g(&["checkout", "-b", "historical"]);
14314        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14315        g(&["add", "."]);
14316        g(&["commit", "-m", "old alternative"]);
14317        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14318        g(&["checkout", "-b", "magi/task-selected", &base]);
14319        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14320        g(&["add", "."]);
14321        g(&["commit", "-m", "measurement infrastructure"]);
14322        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14323        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14324        g(&["add", "."]);
14325        g(&["commit", "-m", "dependent implementation"]);
14326        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14327        g(&["checkout", "main"]);
14328        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14329        g(&["add", "."]);
14330        g(&["commit", "-m", "unrelated base change"]);
14331        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14332        let origin = tmp.path().join("origin.git");
14333        g(&[
14334            "clone",
14335            "--bare",
14336            repo.to_str().unwrap(),
14337            origin.to_str().unwrap(),
14338        ]);
14339        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14340        g(&["fetch", "origin"]);
14341        let selection = crate::pr_recovery::Selection {
14342            url: "https://example.test/project/pull/1".into(),
14343            branch: "magi/task-selected".into(),
14344            head: head.clone(),
14345            source_head: None,
14346            source_run: None,
14347            source_proof: None,
14348            disposition: Some(crate::pr_recovery::Disposition::Review),
14349            instruction: "preserve measured behavior".into(),
14350            approved_at: jiff::Timestamp::now(),
14351        };
14352        let mut template = runner_at(RunStatus::Reviewing);
14353        template.state.repo = repo.clone();
14354        template.state.base_commit = current.clone();
14355        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14356        template.state.origin = Some(crate::run::Origin::queue("selected"));
14357        template.state.pr_head = Some(selection.branch.clone());
14358        template.state.pr_recovery = Some(selection.clone());
14359        let mut runner =
14360            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14361                .await
14362                .unwrap();
14363        let wt = runner.state.winner().unwrap().worktree.clone();
14364        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14365        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14366        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14367        assert!(runner.state.seeds.is_empty());
14368        assert_eq!(runner.state.instruction, "preserve measured behavior");
14369        runner.sync_to_base().await.unwrap();
14370        assert_eq!(
14371            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14372            "prerequisite"
14373        );
14374        assert_eq!(
14375            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14376            "measured implementation"
14377        );
14378        assert_eq!(
14379            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14380            "current main change"
14381        );
14382        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14383        assert_eq!(
14384            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14385            head
14386        );
14387
14388        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14389        let mut partial = runner_at(RunStatus::Reviewing);
14390        partial.state.repo = repo.clone();
14391        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14392        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14393            run: Some("exit 1".into()),
14394            ..Default::default()
14395        }];
14396        let selection = runner.state.pr_recovery.clone().unwrap();
14397        partial.state.pr_recovery = Some(selection.clone());
14398        partial.state.pr_head = Some(selection.branch.clone());
14399        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14400        let id = partial.state.id.clone();
14401        let error = match Runner::open_selected(
14402            &repo,
14403            partial.state,
14404            partial.roles,
14405            1,
14406            runner.state.base_commit.clone(),
14407            selection,
14408        )
14409        .await
14410        {
14411            Ok(_) => panic!("setup must fail"),
14412            Err(e) => e,
14413        };
14414        assert_eq!(
14415            error
14416                .downcast_ref::<crate::run::StartupFailure>()
14417                .unwrap()
14418                .run,
14419            id
14420        );
14421        let failed = RunState::load(&id).unwrap();
14422        assert_eq!(failed.status, RunStatus::Failed);
14423        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14424        assert!(Runner::resume(&id).is_err());
14425        runner.state.status = RunStatus::Ready;
14426        runner.state.driver_exited = true;
14427        runner.state.save().unwrap();
14428        let mut retry = runner_at(RunStatus::Reviewing);
14429        retry.state.repo = repo.clone();
14430        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14431        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14432        reselected.approved_at = jiff::Timestamp::now();
14433        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14434        retry.state.pr_head = Some(reselected.branch.clone());
14435        retry.state.pr_recovery = Some(reselected.clone());
14436        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14437            .await
14438            .unwrap();
14439        let restarted = Runner::open_selected(
14440            &repo,
14441            retry.state,
14442            retry.roles,
14443            1,
14444            runner.state.base_commit.clone(),
14445            reselected,
14446        )
14447        .await
14448        .unwrap();
14449        assert_eq!(
14450            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14451                .await
14452                .unwrap(),
14453            head
14454        );
14455        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14456        assert_eq!(
14457            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14458            head
14459        );
14460    }
14461
14462    #[tokio::test]
14463    #[cfg(unix)]
14464    async fn fallback_and_interrupted_restoration_require_coordination() {
14465        use std::os::fd::AsRawFd as _;
14466        ask_test_home();
14467        let tmp = tempfile::tempdir().unwrap();
14468        let mut runner = registered_runner(tmp.path()).await;
14469        runner.state.config.graph.timeout_resource_wait = 1;
14470        let winner = runner.state.candidates[0].clone();
14471        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14472        let leftover = winner.worktree.join("leftover");
14473        std::fs::write(&leftover, "untrusted edits").unwrap();
14474        let next = tmp.path().join("next-fixer");
14475        let mut failed = spec("first");
14476        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14477        let mut successor = spec("second");
14478        successor.command = vec![
14479            "sh".into(),
14480            "-c".into(),
14481            format!("touch '{}'; echo done", next.display()),
14482        ];
14483        let attempts = vec![
14484            (failed.clone(), "first".into()),
14485            (successor.clone(), "second".into()),
14486        ];
14487        let lock_path = runner.state.repo.join(".git/updater.lock");
14488        let external = std::fs::File::create(&lock_path).unwrap();
14489        assert_eq!(
14490            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14491            0
14492        );
14493        let artifacts = tmp.path().join("artifacts");
14494        let build = |spec, seat| SeatJob {
14495            spec,
14496            seat,
14497            cwd: winner.worktree.clone(),
14498            prompt: "fixture".into(),
14499            timeout: Duration::from_secs(10),
14500            allow_write: true,
14501            sessions: false,
14502            artifacts: artifacts.clone(),
14503            stem: "fix".into(),
14504            handover: None,
14505        };
14506        let (_, _, out) = runner
14507            .ask_fixer_among(
14508                attempts.clone(),
14509                "graft",
14510                None,
14511                Some((&winner.worktree, &head)),
14512                build,
14513            )
14514            .await;
14515        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14516        assert!(leftover.exists(), "contention must prevent clean");
14517        assert!(!next.exists());
14518        assert!(runner.state.handovers.is_empty());
14519        // Early interrupted-repair rollback uses the same protected route.
14520        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14521            from_run: "earlier".into(),
14522            generation: 1,
14523            mode: crate::recovery::Mode::VerifyRepair,
14524            reasons: Vec::new(),
14525            reviewed_head: None,
14526            rounds: Vec::new(),
14527            open_findings: Vec::new(),
14528            evidence: None,
14529            criteria_inherited: true,
14530            repair: crate::recovery::Repair::Started {
14531                before: head.clone(),
14532            },
14533        });
14534        assert!(
14535            runner
14536                .recovery_repair(&winner)
14537                .await
14538                .unwrap_err()
14539                .is::<crate::repo_lock::ResourceBlocked>()
14540        );
14541        assert!(matches!(
14542            runner.state.recovery.as_ref().unwrap().repair,
14543            crate::recovery::Repair::Started { .. }
14544        ));
14545        assert!(leftover.exists());
14546        drop(external);
14547        runner.recovery_repair(&winner).await.unwrap();
14548        assert!(!leftover.exists());
14549        assert!(matches!(
14550            runner.state.recovery.as_ref().unwrap().repair,
14551            crate::recovery::Repair::Done { .. }
14552        ));
14553        // A hard lock-open failure also leaves HEAD and untracked work intact.
14554        std::fs::remove_file(&lock_path).unwrap();
14555        std::fs::create_dir(&lock_path).unwrap();
14556        std::fs::write(&leftover, "untrusted edits").unwrap();
14557        let (_, _, out) = runner
14558            .ask_fixer_among(
14559                attempts,
14560                "graft",
14561                None,
14562                Some((&winner.worktree, &head)),
14563                build,
14564            )
14565            .await;
14566        assert!(matches!(out, AgentOutcome::Failed(_)));
14567        assert!(leftover.exists());
14568        assert!(!next.exists());
14569        assert_eq!(
14570            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14571            head
14572        );
14573    }
14574
14575    #[cfg(unix)]
14576    async fn registered_runner(root: &Path) -> Runner {
14577        repository_runner(root, crate::config::test_coordination()).await
14578    }
14579
14580    async fn repository_runner(
14581        root: &Path,
14582        coordination: Option<crate::config::Coordination>,
14583    ) -> Runner {
14584        ask_test_home();
14585        let repo = root.join("repo");
14586        std::fs::create_dir_all(&repo).unwrap();
14587        init_repo(&repo);
14588        if coordination.is_some() {
14589            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14590        }
14591        let mut runner = runner_at(RunStatus::Prep);
14592        runner.state.repo = repo.clone();
14593        runner.state.config.coordination = coordination;
14594        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14595        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14596        runner.state.config.graph.advise = false;
14597        runner.roles.implementers = vec![spec("implement")];
14598        runner.roles.judges = vec![spec("judge")];
14599        runner.prep().await.unwrap();
14600        runner
14601    }
14602
14603    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14604
14605    #[tokio::test]
14606    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14607        let tmp = tempfile::tempdir().unwrap();
14608        let mut runner = repository_runner(tmp.path(), None).await;
14609        let prompts = runner.state.config.prompts.clone();
14610        let run = runner.state.id.clone();
14611        let ctx = WaveCtx {
14612            carry_seats: false,
14613            pauses: None,
14614            run: &run,
14615            node: "advise",
14616            prompts: &prompts,
14617            cache: None,
14618            round: None,
14619        };
14620        let jobs: Vec<_> = ["alpha", "beta"]
14621            .into_iter()
14622            .map(|id| SeatJob {
14623                spec: scripted(id, AUTH_OUTAGE),
14624                seat: SeatState::new(id, id, 7),
14625                prompt: "propose".into(),
14626                cwd: runner.state.candidates[0].worktree.clone(),
14627                timeout: Duration::from_secs(10),
14628                allow_write: false,
14629                sessions: false,
14630                artifacts: agent::artifacts_dir(&runner.state.dir()),
14631                stem: id.into(),
14632                handover: None,
14633            })
14634            .collect();
14635        let mut losses = Vec::new();
14636        let results = ask_wave_with(
14637            jobs.clone(),
14638            runner.sem.clone(),
14639            2,
14640            &[],
14641            &ctx,
14642            &mut losses,
14643            &mut runner.state,
14644            &|_: &str| Ok(()),
14645        )
14646        .await;
14647        assert!(
14648            results
14649                .iter()
14650                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14651        );
14652        assert_eq!(runner.state.quota.len(), 2);
14653        assert!(
14654            runner
14655                .state
14656                .quota
14657                .iter()
14658                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14659        );
14660        assert!(!runner.state.advise_attempted);
14661        runner.stop_for_resource().unwrap();
14662        assert_eq!(runner.state.status, RunStatus::Blocked);
14663        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14664        assert_eq!(
14665            runner.state.resource_block.as_ref().unwrap().kind,
14666            "agent authentication"
14667        );
14668        // A machine loss followed by an ordinary failure is not evidence
14669        // that the whole node was unavailable, and must not earn a refund.
14670        runner.state.resource_block = None;
14671        runner.state.quota.clear();
14672        let mut mixed = jobs;
14673        mixed[1].spec.command = vec!["false".into()];
14674        let results = ask_wave_with(
14675            mixed,
14676            runner.sem.clone(),
14677            0,
14678            &[],
14679            &ctx,
14680            &mut losses,
14681            &mut runner.state,
14682            &|_: &str| Ok(()),
14683        )
14684        .await;
14685        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14686        assert!(runner.state.resource_block.is_none());
14687        assert_eq!(losses.len(), 1);
14688    }
14689
14690    #[tokio::test]
14691    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14692        let tmp = tempfile::tempdir().unwrap();
14693        let mut runner = repository_runner(tmp.path(), None).await;
14694        let implementer = scripted("implement", AUTH_OUTAGE);
14695        runner.roles.implementers = vec![implementer.clone()];
14696        runner.state.config.agents = vec![implementer];
14697        runner.implement().await.unwrap();
14698        assert_eq!(runner.state.status, RunStatus::Blocked);
14699        assert_eq!(runner.state.quota.len(), 1);
14700        assert_eq!(
14701            runner.state.quota[0].cause,
14702            agent::QuotaCause::Authentication
14703        );
14704        let candidate = &runner.state.candidates[0];
14705        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14706        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14707    }
14708
14709    #[tokio::test]
14710    #[cfg(unix)]
14711    async fn parked_implementer_persists_completed_work() {
14712        for registered in [false, true] {
14713            let tmp = tempfile::tempdir().unwrap();
14714            let mut runner = registered_runner(tmp.path()).await;
14715            if !registered {
14716                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14717            }
14718            let marker = tmp.path().join("started");
14719            let release = tmp.path().join("release");
14720            let mut implementer = spec("implement");
14721            implementer.command = vec![
14722                "sh".into(),
14723                "-c".into(),
14724                format!(
14725                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14726                    marker.display(),
14727                    release.display()
14728                ),
14729            ];
14730            runner.roles.implementers = vec![implementer.clone()];
14731            runner.state.config.agents = vec![implementer];
14732            let pause = runner.pause.clone();
14733            let (result, ()) = tokio::join!(runner.implement(), async {
14734                tokio::time::timeout(Duration::from_secs(10), async {
14735                    while !marker.exists() {
14736                        tokio::time::sleep(Duration::from_millis(10)).await;
14737                    }
14738                })
14739                .await
14740                .unwrap();
14741                pause.park();
14742                std::fs::write(&release, "go").unwrap();
14743            });
14744            result.unwrap();
14745            let candidate = &runner.state.candidates[0];
14746            assert!(candidate.commits > 0, "{candidate:?}");
14747            assert!(candidate.summary.contains("saved-result"));
14748            assert_eq!(candidate.files, 1);
14749            let before = candidate.commits;
14750            runner.implement().await.unwrap();
14751            assert_eq!(runner.state.candidates[0].commits, before);
14752        }
14753    }
14754
14755    #[tokio::test]
14756    #[cfg(unix)]
14757    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14758        let tmp = tempfile::tempdir().unwrap();
14759        let mut runner = registered_runner(tmp.path()).await;
14760        let winner = runner.state.candidates[0].clone();
14761        let cache = tmp.path().join("cache");
14762        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14763        runner.state.config.graph.timeout_resource_wait = 0;
14764        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14765        let crate::cache::AcquireOutcome::Acquired(held) =
14766            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14767        else {
14768            panic!("cache must be free")
14769        };
14770        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14771            from_run: "earlier".into(),
14772            generation: 1,
14773            mode: crate::recovery::Mode::VerifyRepair,
14774            reasons: Vec::new(),
14775            reviewed_head: None,
14776            rounds: Vec::new(),
14777            open_findings: Vec::new(),
14778            evidence: None,
14779            criteria_inherited: true,
14780            repair: crate::recovery::Repair::NotStarted,
14781        });
14782        runner.recovery_repair(&winner).await.unwrap();
14783        assert!(matches!(
14784            runner.state.recovery.as_ref().unwrap().repair,
14785            crate::recovery::Repair::WaitingForRestore { .. }
14786        ));
14787        drop(held);
14788        runner.state.resource_block = None;
14789        runner.state.config.graph.timeout_resource_wait = 1;
14790        runner.recovery_repair(&winner).await.unwrap();
14791        assert!(matches!(
14792            runner.state.recovery.as_ref().unwrap().repair,
14793            crate::recovery::Repair::Done { .. }
14794        ));
14795        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14796    }
14797
14798    #[tokio::test]
14799    #[cfg(unix)]
14800    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14801        use std::os::fd::AsRawFd as _;
14802        for graft in [false, true] {
14803            let tmp = tempfile::tempdir().unwrap();
14804            let mut runner = registered_runner(tmp.path()).await;
14805            runner.state.config.graph.timeout_resource_wait = 1;
14806            let winner = runner.state.candidates[0].clone();
14807            let marker = tmp.path().join("fixer-started");
14808            let mut fixer = spec("fixer");
14809            fixer.command = vec![
14810                "sh".into(),
14811                "-c".into(),
14812                format!("echo started >> '{}'; echo done", marker.display()),
14813            ];
14814            runner.roles.fixer = Some(vec![fixer]);
14815            if graft {
14816                let mut loser = winner.clone();
14817                loser.label = 'B';
14818                loser.index = 1;
14819                runner.state.candidates.push(loser);
14820                runner.state.config.graph.graft_runner_up = true;
14821                runner.state.tally = Some(Tally {
14822                    first_choice: BTreeMap::new(),
14823                    borda: BTreeMap::new(),
14824                    winner: winner.label,
14825                    rankings: 1,
14826                    unanimous_initial: true,
14827                    deliberated: false,
14828                    changed_votes: 0,
14829                    unanimous_final: true,
14830                    tie_break: None,
14831                    judges: 1,
14832                    present: 1,
14833                    quorum: 1,
14834                    met_quorum: true,
14835                    uncontested: None,
14836                });
14837                runner.state.judgements.push(crate::run::Judgement {
14838                    judge: 1,
14839                    seat: "judge-1".into(),
14840                    agent: "judge".into(),
14841                    ranking: vec![winner.label, 'B'],
14842                    reasons: BTreeMap::new(),
14843                    confidence: None,
14844                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14845                    order: vec![0, 1],
14846                    failed: None,
14847                    duration_ms: 0,
14848                });
14849            } else {
14850                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14851                    from_run: "earlier".into(),
14852                    generation: 1,
14853                    mode: crate::recovery::Mode::VerifyRepair,
14854                    reasons: Vec::new(),
14855                    reviewed_head: None,
14856                    rounds: Vec::new(),
14857                    open_findings: Vec::new(),
14858                    evidence: None,
14859                    criteria_inherited: true,
14860                    repair: crate::recovery::Repair::NotStarted,
14861                });
14862            }
14863            let reader = crate::repo_lock::SeatLease::for_checkout(
14864                &winner.worktree,
14865                false,
14866                crate::config::test_coordination().as_ref(),
14867            )
14868            .await
14869            .unwrap();
14870            let pause = runner.pause.clone();
14871            let (result, ()) = tokio::join!(
14872                async {
14873                    if graft {
14874                        runner.graft().await
14875                    } else {
14876                        runner.recovery_repair(&winner).await
14877                    }
14878                },
14879                async {
14880                    tokio::time::sleep(Duration::from_millis(100)).await;
14881                    pause.park();
14882                }
14883            );
14884            result.unwrap();
14885            assert!(!marker.exists(), "a blocked seat must not spawn");
14886            // Exercise the persisted state, as a binary replacement/resume does.
14887            runner.state = RunState::load(&runner.state.id).unwrap();
14888            if graft {
14889                assert_eq!(
14890                    runner.state.graft.as_ref().unwrap().outcome,
14891                    GraftOutcome::WaitingForRestore
14892                );
14893            } else {
14894                assert!(matches!(
14895                    runner.state.recovery.as_ref().unwrap().repair,
14896                    crate::recovery::Repair::WaitingForRestore { .. }
14897                ));
14898            }
14899            drop(reader);
14900            runner.pause = Pause::new();
14901            runner.state.resource_block = None;
14902            // Deferred restoration must still respect an external lock owner.
14903            let external =
14904                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14905            assert_eq!(
14906                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14907                0
14908            );
14909            let leftover = winner.worktree.join("leftover");
14910            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14911            let result = if graft {
14912                runner.graft().await
14913            } else {
14914                runner.recovery_repair(&winner).await
14915            };
14916            assert!(
14917                result
14918                    .unwrap_err()
14919                    .is::<crate::repo_lock::ResourceBlocked>()
14920            );
14921            assert!(leftover.exists());
14922            assert!(!marker.exists());
14923            drop(external);
14924            runner.state.resource_block = None;
14925            let result = if graft {
14926                runner.graft().await
14927            } else {
14928                runner.recovery_repair(&winner).await
14929            };
14930            result.unwrap();
14931            assert!(!leftover.exists(), "restore must precede retry");
14932            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14933            if graft {
14934                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14935            } else {
14936                assert!(matches!(
14937                    runner.state.recovery.as_ref().unwrap().repair,
14938                    crate::recovery::Repair::Done { .. }
14939                ));
14940            }
14941        }
14942    }
14943
14944    #[tokio::test]
14945    #[cfg(unix)]
14946    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14947        use std::os::fd::AsRawFd as _;
14948        let tmp = tempfile::tempdir().unwrap();
14949        let mut runner = registered_runner(tmp.path()).await;
14950        runner.state.config.graph.timeout_resource_wait = 1;
14951        let winner = runner.state.candidates[0].clone();
14952        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14953        runner.state.tally = Some(Tally {
14954            first_choice: BTreeMap::new(),
14955            borda: BTreeMap::new(),
14956            winner: winner.label,
14957            rankings: 0,
14958            unanimous_initial: false,
14959            deliberated: false,
14960            changed_votes: 0,
14961            unanimous_final: false,
14962            tie_break: None,
14963            judges: 0,
14964            present: 0,
14965            quorum: 0,
14966            met_quorum: true,
14967            uncontested: Some("fixture".into()),
14968        });
14969        runner.state.graft = Some(GraftRecord {
14970            pre: pre.clone(),
14971            post: None,
14972            lines: None,
14973            limit: None,
14974            agent: None,
14975            outcome: GraftOutcome::Running,
14976            requested: Vec::new(),
14977            adoption: None,
14978            adoption_error: None,
14979        });
14980        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14981        git::commit_all(&winner.worktree, "unfinished graft")
14982            .await
14983            .unwrap();
14984        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14985        let leftover = winner.worktree.join("leftover");
14986        std::fs::write(&leftover, "untrusted edits").unwrap();
14987        let path = runner.state.repo.join(".git/updater.lock");
14988        let external = std::fs::File::create(&path).unwrap();
14989        assert_eq!(
14990            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14991            0
14992        );
14993        assert!(
14994            runner
14995                .graft()
14996                .await
14997                .unwrap_err()
14998                .is::<crate::repo_lock::ResourceBlocked>()
14999        );
15000        assert!(matches!(
15001            runner.state.graft.as_ref().unwrap().outcome,
15002            GraftOutcome::Running
15003        ));
15004        assert_eq!(
15005            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15006            unfinished
15007        );
15008        assert!(leftover.exists());
15009        drop(external);
15010        std::fs::remove_file(&path).unwrap();
15011        std::fs::create_dir(&path).unwrap();
15012        assert!(runner.graft().await.is_err());
15013        assert!(matches!(
15014            runner.state.graft.as_ref().unwrap().outcome,
15015            GraftOutcome::Running
15016        ));
15017        assert_eq!(
15018            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15019            unfinished
15020        );
15021        assert!(leftover.exists());
15022        std::fs::remove_dir(&path).unwrap();
15023        runner.graft().await.unwrap();
15024        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15025        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15026        assert!(!leftover.exists());
15027    }
15028
15029    #[tokio::test]
15030    #[cfg(unix)]
15031    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15032        use std::os::fd::AsRawFd as _;
15033        let tmp = tempfile::tempdir().unwrap();
15034        let mut runner = registered_runner(tmp.path()).await;
15035        runner.state.prep_complete = Some(false);
15036        runner.state.config.graph.timeout_resource_wait = 1;
15037        runner.state.save().unwrap();
15038        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15039        assert_eq!(
15040            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15041            0
15042        );
15043        runner.execute().await.unwrap();
15044        let disk = RunState::load(&runner.state.id).unwrap();
15045        assert!(disk.driver_exited);
15046        assert_eq!(disk.driver_pid, None);
15047        assert_eq!(runner.state.status, RunStatus::Blocked);
15048        assert_eq!(
15049            runner.state.resource_block.as_ref().unwrap().kind,
15050            "repository coordination"
15051        );
15052        assert!(runner.state.prep_failure.is_none());
15053        assert!(runner.state.seats.is_empty());
15054        runner.pause.park();
15055        // Include headroom for Git subprocesses under parallel test load;
15056        // the repository contention deadline itself remains one second.
15057        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15058            .await
15059            .unwrap()
15060            .unwrap();
15061        assert!(runner.state.parked);
15062        assert!(runner.state.prep_failure.is_none());
15063        let disk = RunState::load(&runner.state.id).unwrap();
15064        assert!(disk.driver_exited);
15065        assert_eq!(disk.driver_pid, None);
15066    }
15067
15068    #[tokio::test]
15069    #[cfg(unix)]
15070    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15071        use std::os::fd::AsRawFd as _;
15072        let tmp = tempfile::tempdir().unwrap();
15073        let mut runner = registered_runner(tmp.path()).await;
15074        runner.state.prep_complete = Some(false);
15075        runner.state.config.graph.timeout_resource_wait = 30;
15076        runner.state.save().unwrap();
15077        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15078        assert_eq!(
15079            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15080            0
15081        );
15082        assert!(
15083            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15084                .await
15085                .is_err()
15086        );
15087        let disk = RunState::load(&runner.state.id).unwrap();
15088        assert!(disk.driver_exited);
15089        assert_eq!(disk.driver_pid, None);
15090        assert!(disk.status.resumable());
15091        assert!(runner.state.driver_exited);
15092        assert_eq!(runner.state.driver_pid, None);
15093        runner.state.save().unwrap();
15094        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15095    }
15096
15097    #[tokio::test]
15098    #[cfg(unix)]
15099    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15100        let tmp = tempfile::tempdir().unwrap();
15101        let mut runner = registered_runner(tmp.path()).await;
15102        let winner = runner.state.candidates[0].clone();
15103        let cache = tmp.path().join("cache");
15104        let started = tmp.path().join("pre-gate-started");
15105        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15106        runner.state.config.verify.gate =
15107            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15108        let home = crate::run::home();
15109        let owner =
15110            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15111        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15112            crate::cache::AcquireOutcome::Acquired(g) => g,
15113            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15114        };
15115        let mut gate = Box::pin(runner.run_gate(&winner));
15116        assert!(
15117            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15118                .await
15119                .is_err()
15120        );
15121        assert!(!started.exists(), "pre-gate must wait for the cache too");
15122        // Simulate the wave that already owns the cache entering invoke.
15123        // It must obtain the editor reservation and finish without waiting
15124        // for the gate that is queued behind its cache.
15125        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15126        let editor = crate::repo_lock::SeatLease::checkout_with(
15127            &winner.worktree,
15128            true,
15129            crate::config::test_coordination().as_ref(),
15130            &wait,
15131        )
15132        .await
15133        .unwrap();
15134        drop(editor);
15135        guard.release();
15136        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15137            .await
15138            .unwrap()
15139            .unwrap();
15140        drop(gate);
15141        assert!(outcomes.iter().all(CommandOutcome::ok));
15142        assert!(started.exists());
15143    }
15144
15145    #[tokio::test]
15146    #[cfg(unix)]
15147    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15148        use std::os::fd::AsRawFd as _;
15149
15150        ask_test_home();
15151        let tmp = tempfile::tempdir().unwrap();
15152        let repo = tmp.path().join("repo");
15153        std::fs::create_dir_all(&repo).unwrap();
15154        init_repo(&repo);
15155        let common = repo.join(".git");
15156        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15157        let mut runner = runner_at(RunStatus::Prep);
15158        runner.state.repo = repo.clone();
15159        runner.state.config.coordination = crate::config::test_coordination();
15160        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15161        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15162        runner.state.config.graph.advise = false;
15163        runner.roles.implementers = vec![spec("implement")];
15164        runner.roles.judges = vec![spec("judge")];
15165        runner.prep().await.unwrap();
15166        let winner = runner.state.candidates[0].clone();
15167        // Emulate edits already left by a completed seat. The fixture CLI
15168        // only reports completion; it performs no repository operations.
15169        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15170        let completed = tmp.path().join("seat-completed");
15171        runner.roles.implementers[0].command = vec![
15172            "sh".into(),
15173            "-c".into(),
15174            format!("touch '{}'; echo completed", completed.display()),
15175        ];
15176        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15177        assert_eq!(
15178            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15179            0
15180        );
15181        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15182        let mut implement = Box::pin(runner.implement());
15183        tokio::select! {
15184            result = &mut implement => panic!("rescue must wait: {result:?}"),
15185            result = tokio::time::timeout(Duration::from_secs(10), async {
15186                while !completed.exists() {
15187                    tokio::time::sleep(Duration::from_millis(20)).await;
15188                }
15189            }) => result.unwrap(),
15190        }
15191        assert!(
15192            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15193                .await
15194                .is_err()
15195        );
15196        assert_eq!(
15197            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15198            before
15199        );
15200        assert_eq!(
15201            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15202            0
15203        );
15204        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15205            .await
15206            .unwrap()
15207            .unwrap();
15208        drop(implement);
15209        assert_ne!(
15210            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15211            before
15212        );
15213        assert!(git::is_clean(&winner.worktree).await.unwrap());
15214
15215        let started = tmp.path().join("pre-gate-started");
15216        let proceed = tmp.path().join("pre-gate-proceed");
15217        let checked = tmp.path().join("gate-ran");
15218        runner.state.config.verify.pre_gate = vec![format!(
15219            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15220            started.display(),
15221            proceed.display()
15222        )];
15223        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15224        assert_eq!(
15225            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15226            0
15227        );
15228        let mut gate = Box::pin(runner.run_gate(&winner));
15229        assert!(
15230            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15231                .await
15232                .is_err()
15233        );
15234        assert!(!started.exists());
15235        assert!(!checked.exists());
15236        assert!(git::is_clean(&winner.worktree).await.unwrap());
15237        assert_eq!(
15238            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15239            0
15240        );
15241        tokio::select! {
15242            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15243            result = tokio::time::timeout(Duration::from_secs(10), async {
15244                while !started.exists() {
15245                    tokio::time::sleep(Duration::from_millis(20)).await;
15246                }
15247            }) => result.unwrap(),
15248        }
15249        assert_eq!(
15250            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15251            -1,
15252            "mechanical edits must hold the real lock"
15253        );
15254        std::fs::write(proceed, "ready").unwrap();
15255        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15256            .await
15257            .unwrap()
15258            .unwrap();
15259        drop(gate);
15260        assert!(outcomes.iter().all(CommandOutcome::ok));
15261        assert!(checked.exists());
15262        assert!(runner.state.pre_gate_commit.is_some());
15263        assert!(git::is_clean(&winner.worktree).await.unwrap());
15264        assert_eq!(
15265            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15266            0,
15267            "gate completion must release coordination"
15268        );
15269        drop(outsider);
15270        std::fs::remove_file(common.join("updater.lock")).unwrap();
15271        std::fs::create_dir(common.join("updater.lock")).unwrap();
15272        std::fs::remove_file(&started).unwrap();
15273        std::fs::remove_file(&checked).unwrap();
15274        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15275        let error = runner.run_gate(&winner).await.unwrap_err();
15276        assert!(
15277            error
15278                .to_string()
15279                .contains("acquire mandatory repository lock")
15280        );
15281        assert!(
15282            !started.exists(),
15283            "failed acquisition must prevent pre-gate"
15284        );
15285        assert!(!checked.exists(), "failed acquisition must prevent gate");
15286        assert!(git::is_clean(&winner.worktree).await.unwrap());
15287        assert_eq!(
15288            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15289            head
15290        );
15291    }
15292
15293    #[tokio::test]
15294    #[cfg(unix)]
15295    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15296        use std::os::fd::AsRawFd as _;
15297        use std::time::Duration;
15298
15299        ask_test_home();
15300        let tmp = tempfile::tempdir().unwrap();
15301        let repo = tmp.path().join("repo");
15302        std::fs::create_dir_all(&repo).unwrap();
15303        init_repo(&repo);
15304        let common = repo.join(".git");
15305        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15306        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15307        assert_eq!(
15308            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15309            0
15310        );
15311        let mut runner = runner_at(RunStatus::Prep);
15312        runner.state.repo = repo.clone();
15313        runner.state.config.coordination = crate::config::test_coordination();
15314        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15315        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15316        runner.state.config.graph.advise = true;
15317        runner.state.config.graph.advisors = 1;
15318        runner.roles.implementers = vec![spec("implement")];
15319        runner.roles.judges = vec![spec("judge")];
15320        let root = runner.state.worktree_root();
15321        let started = tmp.path().join("setup-started");
15322        let proceed = tmp.path().join("setup-proceed");
15323        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15324            run: Some(format!(
15325                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15326                started.display(),
15327                proceed.display()
15328            )),
15329            ..Default::default()
15330        }];
15331        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15332        let config = git::git(&repo, &["config", "--local", "--list"])
15333            .await
15334            .unwrap();
15335        let mut prep = Box::pin(runner.prep());
15336        assert!(
15337            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15338                .await
15339                .is_err()
15340        );
15341        assert!(
15342            !root.exists(),
15343            "contention must prevent even checkout creation"
15344        );
15345        assert!(!started.exists(), "contention must prevent setup");
15346        assert_eq!(
15347            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15348            branches
15349        );
15350        assert_eq!(
15351            git::git(&repo, &["config", "--local", "--list"])
15352                .await
15353                .unwrap(),
15354            config
15355        );
15356        drop(lock);
15357        tokio::select! {
15358            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15359            result = tokio::time::timeout(Duration::from_secs(10), async {
15360                while !started.exists() {
15361                    tokio::time::sleep(Duration::from_millis(20)).await;
15362                }
15363            }) => result.unwrap(),
15364        }
15365        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15366        assert_eq!(
15367            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15368            -1
15369        );
15370        assert!(
15371            tokio::time::timeout(
15372                Duration::from_millis(100),
15373                crate::repo_lock::SeatLease::for_checkout(
15374                    &repo,
15375                    true,
15376                    crate::config::test_coordination().as_ref()
15377                )
15378            )
15379            .await
15380            .is_err(),
15381            "preparation must also exclude local editors"
15382        );
15383        std::fs::write(proceed, "ready").unwrap();
15384        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15385            .await
15386            .unwrap()
15387            .unwrap();
15388        drop(prep);
15389        assert_eq!(runner.state.prep_complete, Some(true));
15390        assert!(root.join("judge-1").exists());
15391        assert!(root.join("advisor-1").exists());
15392        assert_eq!(
15393            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15394            0
15395        );
15396    }
15397
15398    #[tokio::test]
15399    #[cfg(unix)]
15400    async fn preparation_lock_failure_prevents_repository_changes() {
15401        ask_test_home();
15402        let tmp = tempfile::tempdir().unwrap();
15403        let repo = tmp.path().join("repo");
15404        std::fs::create_dir_all(&repo).unwrap();
15405        init_repo(&repo);
15406        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15407        // A directory at the required lock path makes opening the real lock
15408        // fail, rather than merely exercising an invalid registration.
15409        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15410        let mut runner = runner_at(RunStatus::Prep);
15411        runner.state.repo = repo.clone();
15412        runner.state.config.coordination = crate::config::test_coordination();
15413        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15414        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15415        runner.roles.implementers = vec![spec("implement")];
15416        let error = runner.prep().await.unwrap_err();
15417        assert!(
15418            error
15419                .to_string()
15420                .contains("acquire mandatory repository lock")
15421        );
15422        assert!(!runner.state.worktree_root().exists());
15423        assert!(runner.state.candidates.is_empty());
15424        assert!(!repo.join(".git/config.worktree").exists());
15425        assert_eq!(
15426            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15427            "* main"
15428        );
15429    }
15430
15431    #[tokio::test]
15432    async fn preparation_records_creation_intent_before_git_can_fail() {
15433        ask_test_home();
15434        let tmp = tempfile::tempdir().unwrap();
15435        let repo = tmp.path().join("repo");
15436        std::fs::create_dir_all(&repo).unwrap();
15437        init_repo(&repo);
15438        let mut runner = runner_at(RunStatus::Prep);
15439        runner.state.repo = repo.clone();
15440        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15441        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15442        runner.state.config.blind.commit_msg_hook = false;
15443        runner.roles.implementers = vec![spec("one")];
15444        let label = blind::assign_labels(1, runner.state.seed)[0];
15445        let branch = runner.state.branch_for(label);
15446        // A conflicting branch makes Git refuse creation before a candidate
15447        // can be registered. The durable intent must still identify it.
15448        git::git(&repo, &["branch", &branch]).await.unwrap();
15449        assert!(runner.prep().await.is_err());
15450        let saved = RunState::load(&runner.state.id).unwrap();
15451        assert!(saved.candidates.is_empty());
15452        let event = saved
15453            .events
15454            .iter()
15455            .find(|e| e.node == "prep-intent")
15456            .unwrap();
15457        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15458        assert_eq!(intent["branch"], branch);
15459        assert_eq!(intent["start"], runner.state.base_commit);
15460        assert_eq!(
15461            intent["worktree"],
15462            runner
15463                .state
15464                .worktree_root()
15465                .join(format!("cand-{label}"))
15466                .to_string_lossy()
15467                .as_ref()
15468        );
15469        assert_eq!(intent["picks"], serde_json::json!([]));
15470    }
15471
15472    #[tokio::test]
15473    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15474        ask_test_home();
15475        let tmp = tempfile::tempdir().unwrap();
15476        let repo = tmp.path().join("repo");
15477        std::fs::create_dir_all(&repo).unwrap();
15478        init_repo(&repo);
15479        let mut runner = runner_at(RunStatus::Prep);
15480        runner.state.repo = repo.clone();
15481        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15482        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15483        runner.state.sources.pick_commits.push(refs::Input {
15484            token: "missing".into(),
15485            sha: "0".repeat(40),
15486            provenance: refs::Provenance::ExplicitCli,
15487        });
15488        assert!(runner.prep().await.is_err());
15489        assert!(runner.state.candidates.is_empty());
15490        assert!(!runner.state.worktree_root().exists());
15491    }
15492
15493    #[tokio::test]
15494    async fn prose_and_legacy_inference_never_import_into_candidates() {
15495        ask_test_home();
15496        let tmp = tempfile::tempdir().unwrap();
15497        let repo = tmp.path().join("repo");
15498        std::fs::create_dir_all(&repo).unwrap();
15499        init_repo(&repo);
15500        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15501        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15502            .await
15503            .unwrap();
15504        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15505        git::git(&repo, &["add", "."]).await.unwrap();
15506        git::git(&repo, &["commit", "-m", "historical implementation"])
15507            .await
15508            .unwrap();
15509        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15510        git::git(&repo, &["checkout", "main"]).await.unwrap();
15511        for instruction in [
15512            format!("Example: magi/history/A at {source}"),
15513            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15514            format!("Never import magi/history/A or {source}"),
15515            format!("Conversation answer: I mentioned {source} only as background"),
15516        ] {
15517            let mut runner = runner_at(RunStatus::Prep);
15518            runner.state.repo = repo.clone();
15519            runner.state.base_commit = base.clone();
15520            runner.state.instruction = instruction.clone();
15521            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15522            runner.state.config.blind.commit_msg_hook = false;
15523            runner.state.seeds = vec![refs::Seed {
15524                token: "magi/history/A".into(),
15525                sha: source.clone(),
15526                kind: refs::SeedKind::Unmerged,
15527                branch: true,
15528                detail: String::new(),
15529            }];
15530            runner.roles.implementers = vec![spec("one")];
15531            assert_eq!(seeded_instruction(&runner.state), instruction);
15532            // Model an old record with inferred seeds but no structured input field.
15533            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15534            legacy.as_object_mut().unwrap().remove("sources");
15535            runner.state = serde_json::from_value(legacy).unwrap();
15536            assert_eq!(runner.state.sources, refs::Sources::default());
15537            runner.state.save().unwrap();
15538            runner.state = RunState::load(&runner.state.id).unwrap();
15539            runner.prep().await.unwrap();
15540            for candidate in &runner.state.candidates {
15541                assert_eq!(
15542                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15543                    base
15544                );
15545                assert!(!candidate.worktree.join("unwanted").exists());
15546            }
15547        }
15548    }
15549
15550    #[tokio::test]
15551    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15552        ask_test_home();
15553        let tmp = tempfile::tempdir().unwrap();
15554        let repo = tmp.path().join("repo");
15555        std::fs::create_dir_all(&repo).unwrap();
15556        init_repo(&repo);
15557        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15558        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15559        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15560        git::git(&repo, &["add", "."]).await.unwrap();
15561        git::git(&repo, &["commit", "-m", "required seed"])
15562            .await
15563            .unwrap();
15564        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15565        git::git(&repo, &["checkout", "main"]).await.unwrap();
15566        let mut runner = runner_at(RunStatus::Prep);
15567        runner.state.repo = repo.clone();
15568        runner.state.base_commit = base.clone();
15569        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15570        runner.state.config.graph.advise = false;
15571        runner.state.config.blind.commit_msg_hook = false;
15572        runner.state.sources =
15573            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15574                .await
15575                .unwrap();
15576        runner.roles.implementers = vec![spec("one"), spec("two")];
15577        runner.roles.judges = vec![spec("judge")];
15578        let label = blind::assign_labels(2, runner.state.seed)[0];
15579        let branch = runner.state.branch_for(label);
15580        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15581        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15582            .await
15583            .unwrap();
15584        runner.state.candidates.push(Candidate {
15585            index: 0,
15586            label,
15587            agent: "one".into(),
15588            branch,
15589            worktree,
15590            summary: String::new(),
15591            stat: String::new(),
15592            files: 0,
15593            commits: 0,
15594            empty: false,
15595            failed: None,
15596            verified_noop: None,
15597            duration_ms: 0,
15598            folded: false,
15599        });
15600        runner.state.save().unwrap();
15601        // Reload the checkpoint immediately before its first cherry-pick.
15602        runner.state = RunState::load(&runner.state.id).unwrap();
15603        assert_eq!(runner.state.prep_complete, Some(false));
15604        runner.prep().await.unwrap();
15605        assert_eq!(runner.state.prep_complete, Some(true));
15606        assert_eq!(runner.state.candidates.len(), 2);
15607        for candidate in &runner.state.candidates {
15608            assert_eq!(
15609                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15610                git::tree_of(&repo, &seed).await.unwrap()
15611            );
15612            assert_eq!(
15613                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15614                "required commit"
15615            );
15616        }
15617        assert!(runner.state.worktree_root().join("judge-1").exists());
15618        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15619        runner.prep().await.unwrap();
15620        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15621
15622        runner.state.prep_complete = Some(false);
15623        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15624            run: Some(format!(
15625                "test -f '{}'",
15626                tmp.path().join("dependency-ready").display()
15627            )),
15628            ..Default::default()
15629        }];
15630        assert!(runner.prep().await.is_err());
15631        assert!(runner.state.prep_failure.is_some());
15632        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15633        runner.state = RunState::load(&runner.state.id).unwrap();
15634        assert!(
15635            runner.prep().await.is_err(),
15636            "correcting the cause alone must not auto-retry"
15637        );
15638        let failed_at = runner.state.events.last().unwrap().at;
15639        assert!(
15640            runner.retry_preparation(failed_at).is_err(),
15641            "old approvals cannot unlock the hold"
15642        );
15643        let mut task = crate::queue::Task::new(
15644            "retry".into(),
15645            "retry setup".into(),
15646            repo.clone(),
15647            crate::queue::Source::Human,
15648        );
15649        task.start(runner.state.id.clone());
15650        task.hold_machine(Some("setup failed".into()));
15651        task.release_operator("cli", None);
15652        runner
15653            .retry_preparation(task.resumption.as_ref().unwrap().at)
15654            .unwrap();
15655        runner.prep().await.unwrap();
15656        assert!(runner.state.prep_failure.is_none());
15657        assert_eq!(runner.state.prep_complete, Some(true));
15658        assert_eq!(runner.state.candidates.len(), 2);
15659        assert!(runner.state.worktree_root().join("judge-1").exists());
15660        for candidate in &runner.state.candidates {
15661            assert_eq!(
15662                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15663                git::tree_of(&repo, &seed).await.unwrap()
15664            );
15665        }
15666    }
15667
15668    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15669        let mut heads = Vec::new();
15670        for candidate in candidates {
15671            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15672        }
15673        heads
15674    }
15675
15676    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15677        ask_test_home();
15678        let tmp = tempfile::tempdir().unwrap();
15679        let repo = tmp.path().join("repo");
15680        std::fs::create_dir(&repo).unwrap();
15681        init_repo(&repo);
15682        let mut runner = runner_at(RunStatus::Reviewing);
15683        runner.state.repo = repo.clone();
15684        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15685        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15686        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15687            let branch = format!("candidate-{label}");
15688            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15689            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15690                .await
15691                .unwrap();
15692            if label == 'B' {
15693                std::fs::write(worktree.join("change"), "winner").unwrap();
15694                git::rescue_commit(&worktree, "candidate change")
15695                    .await
15696                    .unwrap();
15697            }
15698            runner.state.candidates.push(Candidate {
15699                index,
15700                label,
15701                agent: "alpha".into(),
15702                branch,
15703                worktree,
15704                summary: "completed reply".into(),
15705                stat: String::new(),
15706                files: usize::from(label == 'B'),
15707                commits: usize::from(label == 'B'),
15708                empty: label == 'A',
15709                failed: None,
15710                verified_noop: None,
15711                duration_ms: 1,
15712                folded: false,
15713            });
15714        }
15715        runner.state.judge_skipped = true;
15716        runner.state.tally = Some(Tally {
15717            first_choice: BTreeMap::new(),
15718            borda: BTreeMap::new(),
15719            winner: 'B',
15720            rankings: 0,
15721            unanimous_initial: false,
15722            deliberated: false,
15723            changed_votes: 0,
15724            unanimous_final: false,
15725            tie_break: None,
15726            judges: 0,
15727            present: 0,
15728            quorum: 0,
15729            met_quorum: true,
15730            uncontested: Some("only B produced a change".into()),
15731        });
15732        runner.state.save().unwrap();
15733        (tmp, runner)
15734    }
15735
15736    #[tokio::test]
15737    #[cfg(unix)]
15738    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15739        for (solo, failed_checkpoint) in
15740            [(true, false), (false, false), (true, true), (false, true)]
15741        {
15742            let (_tmp, mut runner) = preservation_resume_fixture().await;
15743            if solo {
15744                runner.state.candidates.retain(|c| c.label == 'A');
15745            }
15746            runner.state.candidates[0].empty = false;
15747            runner.state.candidates[0].summary.clear();
15748            runner.state.tally = None;
15749            runner.state.judge_skipped = false;
15750            runner.state.status = RunStatus::Implementing;
15751            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15752            let mut implementer = spec("alpha");
15753            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15754            runner.roles.implementers = vec![implementer];
15755            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15756            if failed_checkpoint {
15757                std::fs::create_dir_all(&patch).unwrap();
15758            }
15759            runner.state.save().unwrap();
15760            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15761            drop(pressure);
15762            runner.state = RunState::load(&runner.state.id).unwrap();
15763            assert_eq!(runner.state.status, RunStatus::Blocked);
15764            assert!(runner.state.interrupted_implementations.contains(&'A'));
15765            assert_eq!(runner.state.candidates[0].commits, 0);
15766            assert!(!runner.state.candidates[0].empty);
15767            assert!(runner.state.candidates[0].failed.is_none());
15768            if failed_checkpoint {
15769                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15770                std::fs::remove_dir(patch).unwrap();
15771            }
15772            runner.state.resource_block = None;
15773            runner.state.config.disk.min_free_bytes = 0;
15774            runner.roles.implementers[0].command = vec![
15775                "sh".into(),
15776                "-c".into(),
15777                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15778            ];
15779            runner.implement().await.unwrap();
15780            assert!(runner.state.interrupted_implementations.is_empty());
15781            assert!(runner.state.candidate_preservation.is_empty());
15782            assert_eq!(runner.state.candidates[0].commits, 1);
15783            assert!(!runner.state.candidates[0].empty);
15784            assert!(
15785                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15786                    .unwrap()
15787                    .contains("retried empty interrupted seat")
15788            );
15789            if !solo {
15790                assert_eq!(runner.state.candidates[1].commits, 1);
15791            }
15792        }
15793    }
15794
15795    #[tokio::test]
15796    #[cfg(unix)]
15797    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15798        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15799            let (_tmp, mut runner) = preservation_resume_fixture().await;
15800            runner.state.candidates.retain(|c| c.label == 'A');
15801            runner.state.candidates[0].empty = false;
15802            runner.state.candidates[0].summary.clear();
15803            runner.state.tally = None;
15804            runner.state.judge_skipped = false;
15805            runner.state.status = RunStatus::Implementing;
15806            // Force only the real post-spawn watchdog, without failing the
15807            // separate admission check. Existing edits predate its measurement.
15808            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15809            let wt = runner.state.candidates[0].worktree.clone();
15810            std::fs::write(
15811                wt.join("interrupted-source"),
15812                "retain implementation work\n",
15813            )
15814            .unwrap();
15815            if committed {
15816                git::rescue_commit(&wt, "implementation checkpoint")
15817                    .await
15818                    .unwrap();
15819            }
15820            let lock = git::git(
15821                &wt,
15822                &[
15823                    "rev-parse",
15824                    "--path-format=absolute",
15825                    "--git-path",
15826                    "index.lock",
15827                ],
15828            )
15829            .await
15830            .unwrap();
15831            if block_rescue {
15832                std::fs::write(&lock, "fixture-owned lock").unwrap();
15833            }
15834            let mut implementer = spec("alpha");
15835            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15836            runner.roles.implementers = vec![implementer];
15837            runner.state.save().unwrap();
15838            let result = runner.implement().await;
15839            drop(pressure);
15840            assert_eq!(result.is_err(), block_rescue);
15841            assert_eq!(
15842                runner.state.resource_block.as_ref().unwrap().kind,
15843                "disk pressure"
15844            );
15845            assert!(
15846                runner
15847                    .state
15848                    .events
15849                    .iter()
15850                    .any(|e| e.message.contains("agent execution interrupted"))
15851            );
15852            assert!(
15853                runner
15854                    .state
15855                    .dir()
15856                    .join("artifacts/candidate-A-recovery/staged.patch")
15857                    .exists()
15858            );
15859            if block_rescue {
15860                let saved = RunState::load(&runner.state.id).unwrap();
15861                assert!(saved.candidate_preservation.contains_key(&'A'));
15862                assert!(!saved.candidates[0].empty);
15863                assert_eq!(
15864                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15865                    "retain implementation work\n"
15866                );
15867                std::fs::remove_file(lock).unwrap();
15868                runner.state = saved;
15869            } else {
15870                assert_eq!(runner.state.status, RunStatus::Blocked);
15871                assert_eq!(runner.state.candidates[0].commits, 1);
15872            }
15873            // Recovery must save the candidate before deciding whether another
15874            // implementation is needed. No implementer is available on retry.
15875            runner.roles.implementers.clear();
15876            runner.state.config.disk.min_free_bytes = 0;
15877            runner.state.resource_block = None;
15878            runner.implement().await.unwrap();
15879            assert!(runner.state.candidate_preservation.is_empty());
15880            assert_eq!(runner.state.candidates[0].commits, 1);
15881            assert!(!runner.state.candidates[0].empty);
15882            assert!(git::is_clean(&wt).await.unwrap());
15883            assert!(
15884                crate::run::read_artifact(&runner.state, "cand-A.patch")
15885                    .unwrap()
15886                    .contains("retain implementation work")
15887            );
15888        }
15889    }
15890
15891    #[tokio::test]
15892    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15893        let (_tmp, mut runner) = preservation_resume_fixture().await;
15894        // Schema 15 already confirmed the unfolded no-change result.
15895        runner.implement().await.unwrap();
15896        assert!(runner.state.candidate_preservation.is_empty());
15897        assert!(
15898            !runner
15899                .state
15900                .dir()
15901                .join("artifacts/candidate-A-recovery")
15902                .exists()
15903        );
15904        runner.fold_losers().await.unwrap();
15905        assert!(!runner.state.candidates[0].worktree.exists());
15906        for schema in [14, crate::run::SCHEMA] {
15907            runner.state.schema = schema;
15908            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15909            runner.state.save().unwrap();
15910            runner.state = RunState::load(&runner.state.id).unwrap();
15911            runner.implement().await.unwrap();
15912            assert!(runner.state.candidate_preservation.is_empty());
15913            assert!(runner.state.candidates[0].folded);
15914            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15915        }
15916    }
15917
15918    #[tokio::test]
15919    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15920        let (_tmp, mut runner) = preservation_resume_fixture().await;
15921        runner.state.candidates.retain(|c| c.label == 'A');
15922        runner.state.tally = None;
15923        runner.state.judge_skipped = false;
15924        let candidate = &mut runner.state.candidates[0];
15925        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15926        candidate.failed = Some("deployment validation unavailable".into());
15927        std::fs::write(
15928            candidate.worktree.join("repair"),
15929            "future launcher policy\n",
15930        )
15931        .unwrap();
15932        runner
15933            .state
15934            .candidate_preservation
15935            .insert('A', "awaiting persistence".into());
15936        runner.persist_candidates().await.unwrap();
15937        let candidate = &runner.state.candidates[0];
15938        assert!(!candidate.empty);
15939        assert_eq!(candidate.commits, 1);
15940        assert_eq!(candidate.files, 1);
15941        assert!(candidate.failed.is_none());
15942        assert!(candidate.summary.contains("EPERM"));
15943        assert!(
15944            crate::run::read_artifact(&runner.state, "cand-A.patch")
15945                .unwrap()
15946                .contains("future launcher policy")
15947        );
15948        assert!(runner.state.candidate_preservation.is_empty());
15949    }
15950
15951    #[tokio::test]
15952    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15953        let (_tmp, mut runner) = preservation_resume_fixture().await;
15954        let wt = runner.state.candidates[0].worktree.clone();
15955        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15956        runner.state.schema = 14;
15957        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15958        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15959        let tally = serde_json::to_value(&runner.state.tally).unwrap();
15960        for _ in 0..2 {
15961            assert!(runner.implement().await.is_err());
15962            assert_eq!(runner.state.status, RunStatus::Blocked);
15963            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15964            assert!(
15965                runner.state.candidates[0].empty,
15966                "old verdict must remain unusable"
15967            );
15968            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15969            assert!(runner.state.judge_skipped);
15970            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15971            assert!(
15972                crate::run::read_artifact(&runner.state, "cand-A.patch")
15973                    .unwrap()
15974                    .contains("owned edits")
15975            );
15976            assert!(wt.exists());
15977            assert_eq!(
15978                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15979                    .await
15980                    .unwrap(),
15981                1
15982            );
15983            runner.state = RunState::load(&runner.state.id).unwrap();
15984        }
15985    }
15986
15987    #[tokio::test]
15988    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15989        let (_tmp, mut runner) = preservation_resume_fixture().await;
15990        let wt = runner.state.candidates[0].worktree.clone();
15991        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15992        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15993        let index_lock = PathBuf::from(
15994            git::git(
15995                &wt,
15996                &[
15997                    "rev-parse",
15998                    "--path-format=absolute",
15999                    "--git-path",
16000                    "index.lock",
16001                ],
16002            )
16003            .await
16004            .unwrap(),
16005        );
16006        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16007        runner.state.schema = 14;
16008        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16009        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16010        assert!(runner.implement().await.is_err());
16011        assert!(runner.after_implement().is_err());
16012        assert!(runner.fold_losers().await.is_err());
16013        assert_eq!(
16014            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16015            "failed seat edits"
16016        );
16017        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16018        runner.state = RunState::load(&runner.state.id).unwrap();
16019        std::fs::remove_file(index_lock).unwrap();
16020        // Recovery finds changes that the old uncontested result never judged.
16021        assert!(runner.implement().await.is_err());
16022        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16023        assert!(runner.state.candidates[0].empty);
16024        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16025        assert!(
16026            crate::run::read_artifact(&runner.state, "cand-A.patch")
16027                .unwrap()
16028                .contains("failed seat edits")
16029        );
16030        assert!(wt.exists());
16031    }
16032
16033    #[tokio::test]
16034    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16035        let (_tmp, mut runner) = preservation_resume_fixture().await;
16036        let repo = runner.state.repo.clone();
16037        let common = git::git(
16038            &repo,
16039            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16040        )
16041        .await
16042        .unwrap();
16043        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16044        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16045        assert!(runner.fold_losers().await.is_err());
16046        assert!(!runner.state.candidates[0].worktree.exists());
16047        assert!(!runner.state.candidates[0].folded);
16048        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16049        runner.state = RunState::load(&runner.state.id).unwrap();
16050        std::fs::remove_file(branch_lock).unwrap();
16051        let judge = runner.state.worktree_root().join("judge-1");
16052        git::worktree_add_detached(&repo, &judge, "main")
16053            .await
16054            .unwrap();
16055        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16056        runner.roles.judges = vec![spec("judge")];
16057        assert!(runner.fold_losers().await.is_err());
16058        runner.state = RunState::load(&runner.state.id).unwrap();
16059        assert!(
16060            runner.state.candidates[0].folded,
16061            "candidate progress saved before judge failure"
16062        );
16063        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16064        assert!(judge.join("uncommitted").exists());
16065        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16066        runner.fold_losers().await.unwrap();
16067        assert!(!judge.exists());
16068        runner.fold_losers().await.unwrap();
16069    }
16070
16071    #[tokio::test]
16072    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16073        let (_tmp, mut runner) = preservation_resume_fixture().await;
16074        runner.state.candidates.retain(|c| c.label == 'B');
16075        runner.state.tally = None;
16076        runner.state.judge_skipped = false;
16077        runner.state.prep_complete = Some(true);
16078        runner.state.config.graph.advise = false;
16079        runner.state.config.graph.selected_implementer = Some("alpha".into());
16080        let wt = runner.state.candidates[0].worktree.clone();
16081        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16082        runner
16083            .state
16084            .candidate_preservation
16085            .insert('B', "awaiting persistence".into());
16086        let index_lock = git::git(
16087            &wt,
16088            &[
16089                "rev-parse",
16090                "--path-format=absolute",
16091                "--git-path",
16092                "index.lock",
16093            ],
16094        )
16095        .await
16096        .unwrap();
16097        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16098        runner.state.save().unwrap();
16099        assert!(runner.execute_graph().await.is_err());
16100        assert_eq!(runner.state.status, RunStatus::Blocked);
16101        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16102        std::fs::remove_file(index_lock).unwrap();
16103        // Park at the next graph boundary: this tests the real selected-seat
16104        // hold guard without dispatching reviewers or publishing anything.
16105        for _ in 0..2 {
16106            runner.state = RunState::load(&runner.state.id).unwrap();
16107            runner.pause.park();
16108            runner.execute_graph().await.unwrap();
16109            assert_eq!(runner.state.status, RunStatus::Implementing);
16110            assert!(runner.state.candidate_preservation.is_empty());
16111            assert_eq!(runner.state.candidates[0].commits, 2);
16112            assert!(runner.state.tally.is_none());
16113        }
16114        runner.implement().await.unwrap();
16115        assert_eq!(runner.state.status, RunStatus::Judging);
16116        // A genuine selected-implementer hold must still require release.
16117        runner.state.status = RunStatus::Blocked;
16118        runner.state.candidates[0].failed = Some("implementation failed".into());
16119        runner.state.save().unwrap();
16120        let error = runner.execute_graph().await.unwrap_err();
16121        assert!(
16122            error
16123                .to_string()
16124                .contains("selected implementation is held")
16125        );
16126    }
16127
16128    #[tokio::test]
16129    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16130        let (_tmp, mut runner) = preservation_resume_fixture().await;
16131        let repo = runner.state.repo.clone();
16132        let candidate = runner.state.candidates[0].clone();
16133        // Remove only the disposable fixture directory, leaving Git's linked
16134        // worktree registration and branch intact, as in a vanished checkout.
16135        let registered = git::worktree_holding(&repo, &candidate.branch)
16136            .await
16137            .unwrap()
16138            .unwrap();
16139        assert_eq!(
16140            registered.canonicalize().unwrap(),
16141            candidate.worktree.canonicalize().unwrap()
16142        );
16143        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16144        assert_eq!(
16145            git::worktree_holding(&repo, &candidate.branch)
16146                .await
16147                .unwrap(),
16148            Some(registered)
16149        );
16150        assert!(
16151            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16152            "the stale registration initially keeps the branch checked out"
16153        );
16154        runner.fold_losers().await.unwrap();
16155        assert!(runner.state.candidates[0].folded);
16156        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16157        assert_eq!(
16158            git::worktree_holding(&repo, &candidate.branch)
16159                .await
16160                .unwrap(),
16161            None
16162        );
16163        runner.state = RunState::load(&runner.state.id).unwrap();
16164        runner.fold_losers().await.unwrap();
16165    }
16166
16167    #[tokio::test]
16168    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16169        let (_tmp, mut runner) = preservation_resume_fixture().await;
16170        let repo = runner.state.repo.clone();
16171        let candidate = runner.state.candidates[0].clone();
16172        let winner = runner.state.candidates[1].clone();
16173        let registered = git::worktree_holding(&repo, &candidate.branch)
16174            .await
16175            .unwrap()
16176            .unwrap();
16177        assert_eq!(
16178            registered.canonicalize().unwrap(),
16179            candidate.worktree.canonicalize().unwrap()
16180        );
16181        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16182        assert_eq!(
16183            git::worktree_holding(&repo, &candidate.branch)
16184                .await
16185                .unwrap(),
16186            Some(registered)
16187        );
16188        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16189
16190        for _ in 0..2 {
16191            fold_run(&mut runner.state, false, &crate::run::home())
16192                .await
16193                .unwrap();
16194            runner.state = RunState::load(&runner.state.id).unwrap();
16195            assert!(runner.state.candidates[0].folded);
16196            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16197            assert_eq!(
16198                git::worktree_holding(&repo, &candidate.branch)
16199                    .await
16200                    .unwrap(),
16201                None
16202            );
16203            assert!(!runner.state.candidates[1].folded);
16204            assert!(winner.worktree.exists());
16205            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16206        }
16207    }
16208
16209    #[tokio::test]
16210    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16211        ask_test_home();
16212        let tmp = tempfile::tempdir().unwrap();
16213        let repo = tmp.path().join("repo");
16214        std::fs::create_dir(&repo).unwrap();
16215        init_repo(&repo);
16216        let mut runner = runner_at(RunStatus::Implementing);
16217        runner.state.repo = repo.clone();
16218        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16219        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16220        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16221            let branch = format!("candidate-{label}");
16222            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16223            git::worktree_add_branch(&repo, &wt, &branch, "main")
16224                .await
16225                .unwrap();
16226            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16227            git::git(&wt, &["add", "README.md"]).await.unwrap();
16228            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16229            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16230            runner.state.candidates.push(Candidate {
16231                index,
16232                label,
16233                agent: "alpha".to_owned(),
16234                branch,
16235                worktree: wt,
16236                summary: "completed implementation".to_owned(),
16237                stat: String::new(),
16238                files: 0,
16239                commits: 0,
16240                empty: false,
16241                failed: None,
16242                verified_noop: None,
16243                duration_ms: 42,
16244                folded: false,
16245            });
16246            runner
16247                .state
16248                .candidate_preservation
16249                .insert(label, "awaiting persistence".to_owned());
16250        }
16251        let wt = runner.state.candidates[0].worktree.clone();
16252        let index_lock = git::git(
16253            &wt,
16254            &[
16255                "rev-parse",
16256                "--path-format=absolute",
16257                "--git-path",
16258                "index.lock",
16259            ],
16260        )
16261        .await
16262        .unwrap();
16263        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16264        runner.state.save().unwrap();
16265        assert!(runner.persist_candidates().await.is_err());
16266        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16267        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16268        assert_eq!(
16269            runner.state.candidates[1].commits, 1,
16270            "other results are still collected"
16271        );
16272        assert!(
16273            !runner.state.candidates[0].empty,
16274            "failed rescue is not no change"
16275        );
16276        assert_eq!(
16277            runner.state.candidates[0].summary,
16278            "completed implementation"
16279        );
16280        assert!(runner.after_implement().is_err());
16281        let home = tmp.path().join("fold-home");
16282        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16283        assert!(wt.join("new-file").exists());
16284        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16285        assert!(recovery.join("staged.patch").exists());
16286        assert!(recovery.join("unstaged.patch").exists());
16287        assert_eq!(
16288            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16289            "untracked edits\n"
16290        );
16291        runner.state.begin_pass();
16292        runner.state.end_pass_parked(ParkCause::Upgrade {
16293            from: "1.0".into(),
16294            to: Some("1.1".into()),
16295        });
16296        runner.state.save().unwrap();
16297        runner.state = RunState::load(&runner.state.id).unwrap();
16298        assert!(matches!(
16299            runner.state.passes.last().unwrap().end,
16300            Some(crate::run::PassEnd::Parked {
16301                cause: ParkCause::Upgrade { .. }
16302            })
16303        ));
16304        std::fs::remove_file(index_lock).unwrap();
16305        // No implementer is configured: a persistence retry cannot dispatch a seat.
16306        runner.implement().await.unwrap();
16307        assert!(runner.state.candidate_preservation.is_empty());
16308        assert!(
16309            runner
16310                .state
16311                .candidates
16312                .iter()
16313                .all(|c| c.commits == 1 && !c.empty)
16314        );
16315        assert!(git::is_clean(&wt).await.unwrap());
16316        assert!(
16317            crate::run::read_artifact(&runner.state, "cand-A.patch")
16318                .unwrap()
16319                .contains("later tracked edits")
16320        );
16321    }
16322
16323    #[tokio::test]
16324    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16325        ask_test_home();
16326        let tmp = tempfile::tempdir().unwrap();
16327        let repo = tmp.path().join("repo");
16328        std::fs::create_dir(&repo).unwrap();
16329        init_repo(&repo);
16330        let wt = tmp.path().join("candidate");
16331        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16332            .await
16333            .unwrap();
16334        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16335        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16336        assert!(wt.join("uncommitted").exists());
16337        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16338        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16339        let mut runner = runner_at(RunStatus::Implementing);
16340        runner.state.repo = repo.clone();
16341        runner.state.base_commit = "missing-base".to_owned();
16342        runner.state.candidates.push(Candidate {
16343            index: 0,
16344            label: 'A',
16345            agent: "alpha".into(),
16346            branch: "candidate".into(),
16347            worktree: wt.clone(),
16348            summary: "completed".into(),
16349            stat: String::new(),
16350            files: 0,
16351            commits: 0,
16352            empty: false,
16353            failed: None,
16354            verified_noop: None,
16355            duration_ms: 1,
16356            folded: false,
16357        });
16358        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16359        assert!(
16360            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16361                .await
16362                .is_err()
16363        );
16364        assert!(!runner.state.candidates[0].folded);
16365        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16366        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16367        runner
16368            .state
16369            .candidate_preservation
16370            .insert('A', "pending".into());
16371        assert!(runner.persist_candidates().await.is_err());
16372        assert!(!runner.state.candidates[0].empty);
16373        assert!(!runner.state.candidates[0].folded);
16374        assert!(wt.exists());
16375        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16376        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16377        std::fs::create_dir_all(&patch_path).unwrap();
16378        assert!(
16379            runner.persist_candidates().await.is_err(),
16380            "artifact failure stays pending"
16381        );
16382        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16383        assert!(!runner.state.candidates[0].empty);
16384        std::fs::remove_dir(patch_path).unwrap();
16385        runner.persist_candidates().await.unwrap();
16386        assert!(
16387            runner.state.candidates[0].empty,
16388            "only successful inspection confirms no change"
16389        );
16390    }
16391
16392    fn init_repo(dir: &Path) {
16393        let run = |args: &[&str]| {
16394            let out = std::process::Command::new("git")
16395                .args(args)
16396                .current_dir(dir)
16397                .quiet()
16398                .output()
16399                .expect("spawn git");
16400            assert!(
16401                out.status.success(),
16402                "git {args:?} failed: {}",
16403                String::from_utf8_lossy(&out.stderr)
16404            );
16405        };
16406        run(&["init", "-b", "main"]);
16407        run(&["config", "user.name", "magi test"]);
16408        run(&["config", "user.email", "magi@example.com"]);
16409        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16410        run(&["add", "-A"]);
16411        run(&["commit", "-m", "init"]);
16412    }
16413
16414    // `settle_questions` is what closes the ghost the phone showed: a run's
16415    // seat asked something, the run then ended, and nothing was left to
16416    // abandon the question it left `open`. `HOME` is a process-wide
16417    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16418    // first time it runs in the binary — every test below still reaches the
16419    // same directory whichever call won, and each gets its own run id from
16420    // `RunState::new`, so they never collide there.
16421    fn ask_test_home() {
16422        crate::run::pin_test_home();
16423    }
16424
16425    #[test]
16426    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16427        ask_test_home();
16428        let mut runner = runner_at(RunStatus::Prep);
16429        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16430        // A blocked temporary output path makes the terminal save fail while
16431        // preserving the already published reservation.
16432        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16433        let error = guard
16434            .finish(
16435                &mut runner.state,
16436                Err(
16437                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16438                        .into(),
16439                ),
16440            )
16441            .unwrap_err();
16442        assert_eq!(
16443            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16444            std::io::ErrorKind::PermissionDenied
16445        );
16446        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16447        let message = format!("{error:#}");
16448        assert!(message.contains("original refusal"));
16449        assert!(!message.contains("could not save terminal"));
16450        assert_eq!(
16451            error.chain().count(),
16452            2,
16453            "only startup attribution wraps the original error"
16454        );
16455        let saved = RunState::load(&runner.state.id).unwrap();
16456        assert_eq!(saved.status, RunStatus::Blocked);
16457        assert!(!saved.startup.unwrap().complete);
16458    }
16459
16460    #[test]
16461    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16462        ask_test_home();
16463        let mut runner = runner_at(RunStatus::Prep);
16464        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16465        let id = runner.state.id.clone();
16466        drop(guard);
16467        let saved = RunState::load(&id).unwrap();
16468        assert_eq!(saved.status, RunStatus::Blocked);
16469        assert!(!saved.startup.as_ref().unwrap().complete);
16470        assert!(
16471            saved
16472                .startup
16473                .unwrap()
16474                .failure
16475                .unwrap()
16476                .contains("interrupted")
16477        );
16478        assert!(Runner::resume(&id).is_err());
16479    }
16480
16481    #[test]
16482    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16483        ask_test_home();
16484        let mut runner = runner_at(RunStatus::Prep);
16485        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16486        let error = guard
16487            .finish(
16488                &mut runner.state,
16489                Err(
16490                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16491                        .into(),
16492                ),
16493            )
16494            .unwrap_err();
16495        assert_eq!(
16496            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16497            std::io::ErrorKind::PermissionDenied
16498        );
16499        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16500        let mut task = crate::queue::Task::new(
16501            "review".into(),
16502            "criteria".into(),
16503            runner.state.repo.clone(),
16504            crate::queue::Source::Human,
16505        );
16506        task.record_startup_failure(failure.clone());
16507        task.record_startup_failure(failure.clone());
16508        assert_eq!(task.startup_failures.len(), 1);
16509        assert!(task.runs.is_empty());
16510        assert_eq!(task.attempts, 0);
16511    }
16512
16513    #[cfg(unix)]
16514    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16515        ask_test_home();
16516        let tmp = tempfile::tempdir().unwrap();
16517        let repo = tmp.path().join("repo");
16518        std::fs::create_dir(&repo).unwrap();
16519        init_repo(&repo);
16520        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16521        let origin = tmp.path().join("origin.git");
16522        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16523            .await
16524            .unwrap();
16525        git::git(
16526            &repo,
16527            &["remote", "add", "origin", origin.to_str().unwrap()],
16528        )
16529        .await
16530        .unwrap();
16531        git::git(&repo, &["checkout", "-b", "fix/retained"])
16532            .await
16533            .unwrap();
16534        std::fs::write(repo.join("retained"), "retained change").unwrap();
16535        git::git(&repo, &["add", "."]).await.unwrap();
16536        git::git(&repo, &["commit", "-m", "retained implementation"])
16537            .await
16538            .unwrap();
16539        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16540        git::git(&repo, &["checkout", "main"]).await.unwrap();
16541        std::fs::write(repo.join("base-change"), "new base").unwrap();
16542        git::git(&repo, &["add", "."]).await.unwrap();
16543        git::git(&repo, &["commit", "-m", "base moved"])
16544            .await
16545            .unwrap();
16546        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16547        git::git(&repo, &["checkout", "fix/retained"])
16548            .await
16549            .unwrap();
16550        let mut runner = runner_at(RunStatus::Voting);
16551        runner.state.repo = repo.clone();
16552        runner.state.base_commit = base;
16553        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16554        runner.state.config.disk.min_free_bytes = 0;
16555        runner.state.config.graph.review_rounds = 1;
16556        runner.state.config.verify.e2e = vec!["true".into()];
16557        runner.state.config.verify.gate = vec!["true".into()];
16558        runner.state.candidates = vec![Candidate {
16559            index: 0,
16560            label: 'A',
16561            agent: "fixture".into(),
16562            branch: "fix/retained".into(),
16563            worktree: repo.clone(),
16564            summary: String::new(),
16565            stat: String::new(),
16566            files: 1,
16567            commits: 1,
16568            empty: false,
16569            failed: None,
16570            verified_noop: None,
16571            duration_ms: 0,
16572            folded: false,
16573        }];
16574        runner.state.tally = Some(Tally {
16575            first_choice: BTreeMap::new(),
16576            borda: BTreeMap::new(),
16577            winner: 'A',
16578            rankings: 0,
16579            unanimous_initial: true,
16580            deliberated: false,
16581            changed_votes: 0,
16582            unanimous_final: true,
16583            tie_break: None,
16584            judges: 0,
16585            present: 0,
16586            quorum: 0,
16587            met_quorum: true,
16588            uncontested: Some("review-only".into()),
16589        });
16590        let mut task = crate::queue::Task::new(
16591            "fixture".into(),
16592            "fixture".into(),
16593            repo,
16594            crate::queue::Source::Human,
16595        );
16596        task.link_run(&runner.state.id);
16597        crate::queue::Queue::open().put(&mut task).unwrap();
16598        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16599        runner.state.base_sync = Some(BaseSync {
16600            tip: String::new(),
16601            behind: 1,
16602            attempts: BASE_SYNC_ROUNDS,
16603            conflict: None,
16604            already_in: None,
16605        });
16606        runner.sync_to_base().await.unwrap();
16607        assert_eq!(runner.state.status, RunStatus::Blocked);
16608        (tmp, runner, head, task.id)
16609    }
16610
16611    #[cfg(unix)]
16612    #[tokio::test]
16613    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16614        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16615        if std::env::var_os(CHILD).is_none() {
16616            let home = tempfile::tempdir().unwrap();
16617            let output = std::process::Command::new(std::env::current_exe().unwrap())
16618                .args([
16619                    "--exact",
16620                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16621                    "--nocapture",
16622                ])
16623                .env(CHILD, "1")
16624                .env("MAGI_HOME", home.path())
16625                .env("MAGI_WORKER_HOME", home.path())
16626                .env("MAGI_WORKER_ID", "abcdef")
16627                .output()
16628                .unwrap();
16629            assert!(
16630                output.status.success()
16631                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16632                "{}\n{}",
16633                String::from_utf8_lossy(&output.stdout),
16634                String::from_utf8_lossy(&output.stderr)
16635            );
16636            return;
16637        }
16638        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16639        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16640        let home = crate::run::home();
16641        let queue = crate::queue::Queue::open();
16642        let mut owner = queue.get(&task).unwrap();
16643        owner.release_operator("test-release", None);
16644        owner.start(runner.state.id.clone());
16645        queue.put(&mut owner).unwrap();
16646        runner.state.driver_exited = true;
16647        runner.state.task_instruction = Some(owner.instruction.clone());
16648        runner.state.save().unwrap();
16649        let _claim = queue.claim(&task).unwrap();
16650        let reserve = |id: &str| {
16651            let directory = home.join("workers/jobs").join(id);
16652            std::fs::create_dir_all(&directory).unwrap();
16653            let job = crate::worker::Job {
16654                id: id.into(),
16655                sequence: 1,
16656                kind: "task".into(),
16657                key: format!("task:{task}"),
16658                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16659                generation: "fixture-generation".into(),
16660                version: None,
16661                status: crate::worker::Status::Running,
16662                result: None,
16663                pid: Some(std::process::id()),
16664                created_at: 0,
16665                finished_at: None,
16666            };
16667            std::fs::write(
16668                directory.join("job.json"),
16669                serde_json::to_vec(&job).unwrap(),
16670            )
16671            .unwrap();
16672            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16673        };
16674        let _own = reserve("abcdef");
16675        let other = reserve("fedcba");
16676        assert!(
16677            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16678                .await
16679                .is_err()
16680        );
16681        assert!(
16682            RunState::load(&runner.state.id)
16683                .unwrap()
16684                .base_sync_recovery
16685                .authorizations
16686                .is_empty()
16687        );
16688        drop(other);
16689        assert!(crate::daemon::is_working_on(
16690            &home,
16691            &runner.state.id,
16692            Timestamp::now()
16693        ));
16694        assert!(
16695            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16696        );
16697        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16698            .await
16699            .unwrap();
16700        assert!(
16701            RunState::load(&runner.state.id)
16702                .unwrap()
16703                .base_recovery_pending()
16704        );
16705    }
16706
16707    #[cfg(unix)]
16708    #[tokio::test]
16709    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16710        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16711        let home = crate::run::home();
16712        runner.state.driver_pid = Some(std::process::id());
16713        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16714        runner.state.driver_exited = false;
16715        runner.state.begin_pass();
16716        runner.state.save().unwrap();
16717        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16718        driver.record(&runner.state).unwrap();
16719        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16720        // Preserve the durable claim as if the process died before Drop saved.
16721        drop(driver);
16722        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16723        assert_eq!(
16724            runner.state.liveness_under(&home, false),
16725            crate::run::Liveness::Dead
16726        );
16727        let queue = crate::queue::Queue::open();
16728        let _claim = queue.claim(&task).unwrap();
16729        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16730            .await
16731            .unwrap();
16732        let disk = RunState::load(&runner.state.id).unwrap();
16733        assert!(!disk.driver_exited);
16734        assert!(disk.base_recovery_pending());
16735    }
16736
16737    #[cfg(unix)]
16738    #[tokio::test]
16739    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16740        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16741        if std::env::var_os(CHILD).is_none() {
16742            use std::os::unix::fs::PermissionsExt;
16743            let tmp = tempfile::tempdir().unwrap();
16744            let gh = tmp.path().join("gh");
16745            std::fs::write(&gh, r#"#!/bin/sh
16746case "$1 $2" in
16747  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16748  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16749  *) exit 2 ;;
16750esac
16751"#).unwrap();
16752            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16753            let mut paths = vec![tmp.path().to_path_buf()];
16754            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16755            let output = std::process::Command::new(std::env::current_exe().unwrap())
16756                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16757                .env(CHILD, "1")
16758                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16759                .env("PATH", std::env::join_paths(paths).unwrap())
16760                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16761                .output().unwrap();
16762            assert!(
16763                output.status.success()
16764                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16765                "{}\n{}",
16766                String::from_utf8_lossy(&output.stdout),
16767                String::from_utf8_lossy(&output.stderr)
16768            );
16769            return;
16770        }
16771        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16772        let repo = runner.state.repo.clone();
16773        git::git(&repo, &["checkout", "main"]).await.unwrap();
16774        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16775        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16776        git::git(&repo, &["checkout", "fix/retained"])
16777            .await
16778            .unwrap();
16779        runner.state.pr = Some(crate::run::PrRecord {
16780            url: "https://example.test/project/pull/1".into(),
16781            number: 1,
16782            state: "open".into(),
16783            checks: "unknown".into(),
16784            round: 0,
16785            rounds: 1,
16786            red_at_merge: Vec::new(),
16787        });
16788        runner.sync_to_base().await.unwrap();
16789        assert_eq!(runner.state.status, RunStatus::Blocked);
16790        assert_eq!(
16791            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16792            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16793        );
16794        assert!(runner.state.base_sync_closure_pending());
16795        assert!(crate::handover::can_resume(&runner.state));
16796        let queue = crate::queue::Queue::open();
16797        assert!(!crate::base_sync_recovery::held_task(
16798            &queue,
16799            &queue.get(&task).unwrap()
16800        ));
16801        assert!(runner.state.authorize_base_sync(&task).is_err());
16802        // Legacy closure stops have no typed stop but must retain the checkpoint.
16803        runner.state.base_sync_recovery.stop = None;
16804        runner.state.status = RunStatus::Voting;
16805        runner.state.gate_ran = true;
16806        runner.state.config.graph.review_rounds = 0;
16807        runner.state.save().unwrap();
16808        assert!(crate::handover::can_resume(&runner.state));
16809        runner.state = RunState::load(&runner.state.id).unwrap();
16810        runner.execute_graph().await.unwrap();
16811        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16812        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16813        assert_eq!(
16814            runner.state.base_sync.as_ref().unwrap().attempts,
16815            BASE_SYNC_ROUNDS
16816        );
16817        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16818        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16819        assert!(runner.state.reviews.is_empty());
16820        assert!(runner.state.gate.is_empty());
16821    }
16822
16823    #[cfg(unix)]
16824    #[tokio::test]
16825    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16826        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16827        runner.state.driver_exited = true;
16828        runner.state.save().unwrap();
16829        let home = crate::run::home();
16830        let queue = crate::queue::Queue::open();
16831        let _claim = queue.claim(&task).unwrap();
16832        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16833        assert!(
16834            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16835                .await
16836                .is_err()
16837        );
16838        drop(lease);
16839        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16840        assert!(
16841            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16842                .await
16843                .is_err()
16844        );
16845        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16846        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16847            .await
16848            .unwrap();
16849        let disk = RunState::load(&runner.state.id).unwrap();
16850        assert!(disk.base_recovery_pending());
16851        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16852        assert!(
16853            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16854                .await
16855                .is_err()
16856        );
16857        assert_eq!(
16858            RunState::load(&runner.state.id)
16859                .unwrap()
16860                .base_sync_recovery
16861                .authorizations
16862                .len(),
16863            1
16864        );
16865        let mut owner = queue.get(&task).unwrap();
16866        owner.runs.push("20260101-000000-ffff".into());
16867        queue.put(&mut owner).unwrap();
16868        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16869            .await
16870            .unwrap_err();
16871        assert!(error.to_string().contains("run task ownership changed"));
16872    }
16873
16874    #[cfg(unix)]
16875    #[tokio::test]
16876    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16877        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16878        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16879        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16880        legacy["status"] = serde_json::json!("voting");
16881        runner.state = serde_json::from_value(legacy).unwrap();
16882        runner.state.save().unwrap();
16883        for _ in 0..3 {
16884            runner.execute_graph().await.unwrap();
16885            assert_eq!(runner.state.status, RunStatus::Blocked);
16886            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16887            assert!(!crate::handover::can_resume(&runner.state));
16888            assert_eq!(
16889                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16890                head
16891            );
16892            assert!(runner.state.reviews.is_empty());
16893            assert!(!runner.state.gate_ran);
16894        }
16895        let notice = crate::notices::run_ended(&runner.state).unwrap();
16896        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16897        runner.state.begin_pass();
16898        runner.state.status = RunStatus::Voting;
16899        runner.deliberate().await.unwrap();
16900        runner.vote().await.unwrap();
16901        runner.finish_after_tally().await.unwrap();
16902        assert_eq!(runner.state.status, RunStatus::Blocked);
16903        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16904        assert_eq!(
16905            crate::notices::run_ended(&runner.state).unwrap().since,
16906            notice.since
16907        );
16908    }
16909
16910    #[cfg(unix)]
16911    #[tokio::test]
16912    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16913        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16914        for detail in [
16915            "fix/retained conflicts with origin/main and did not rebase: conflict",
16916            "rebased fix/retained locally but origin refused the push",
16917            "unknown failure",
16918        ] {
16919            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16920            runner.state.record_base_sync_stop(&head);
16921            assert!(runner.state.authorize_base_sync(&task).is_err());
16922            runner.state.status = RunStatus::Voting;
16923            runner.execute_graph().await.unwrap();
16924            assert_eq!(runner.state.status, RunStatus::Blocked);
16925            assert!(!crate::handover::can_resume(&runner.state));
16926            assert_eq!(
16927                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16928                head
16929            );
16930        }
16931    }
16932
16933    #[cfg(unix)]
16934    #[tokio::test]
16935    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16936        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16937        runner.state.authorize_base_sync(&task).unwrap();
16938        assert!(runner.state.consume_base_recovery(&head));
16939        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16940        runner.state.reserve_base_operation(&head, &tip);
16941        runner.state.save().unwrap();
16942        runner.state = RunState::load(&runner.state.id).unwrap();
16943        assert!(!runner.state.consume_base_recovery(&head));
16944        runner.sync_to_base().await.unwrap();
16945        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16946        assert_eq!(runner.state.base_episode_used(), 1);
16947        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16948        assert!(
16949            runner
16950                .state
16951                .base_sync_recovery
16952                .operations
16953                .last()
16954                .unwrap()
16955                .finished
16956        );
16957        assert!(!runner.state.base_sync_blocked());
16958        runner.sync_to_base().await.unwrap();
16959        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16960    }
16961
16962    #[cfg(unix)]
16963    #[tokio::test]
16964    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
16965        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16966        // Both historical diagnostic spellings retain an exhaustion identity.
16967        let original = runner
16968            .state
16969            .base_sync
16970            .as_ref()
16971            .unwrap()
16972            .conflict
16973            .clone()
16974            .unwrap();
16975        runner.state.base_sync.as_mut().unwrap().conflict = Some(
16976            original
16977                .replace("synchronization(s)", "rebase(s)")
16978                .replace("synchronizing", "rebasing"),
16979        );
16980        runner.state.base_sync_recovery = Default::default();
16981        runner.state.record_base_sync_stop(&head);
16982        assert_eq!(
16983            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16984            crate::base_sync_recovery::StopKind::Exhausted
16985        );
16986        runner.state.authorize_base_sync(&task).unwrap();
16987        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16988        let auth = legacy["base_sync_recovery"]["authorizations"][0]
16989            .as_object_mut()
16990            .unwrap();
16991        auth.remove("interval");
16992        auth.remove("scope");
16993        runner.state = serde_json::from_value(legacy).unwrap();
16994        assert!(runner.state.consume_base_recovery(&head));
16995        assert_eq!(runner.state.base_episode_used(), 3);
16996        runner.sync_to_base().await.unwrap();
16997        assert_eq!(runner.state.base_episode_used(), 4);
16998        runner.state.resource_block = Some(crate::run::ResourceBlock {
16999            node: "gate".into(),
17000            kind: "cache".into(),
17001            detail: "legacy wait".into(),
17002            at: Timestamp::now(),
17003            cause: crate::run::ResourceBlockCause::Unrecorded,
17004        });
17005        runner.state.save().unwrap();
17006        assert!(
17007            runner
17008                .state
17009                .base_sync_recovery
17010                .episodes
17011                .last()
17012                .unwrap()
17013                .boundary
17014                .is_none()
17015        );
17016        runner.state.resource_block = None;
17017        advance_base_fixture(&runner).await;
17018        runner.sync_to_base().await.unwrap();
17019        assert!(runner.state.base_sync_blocked());
17020        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17021    }
17022
17023    #[cfg(unix)]
17024    async fn advance_base_fixture(runner: &Runner) {
17025        let repo = &runner.state.repo;
17026        for args in [
17027            vec!["checkout", "main"],
17028            vec!["commit", "--allow-empty", "-m", "base advanced"],
17029            vec!["push", "origin", "main"],
17030            vec!["checkout", "fix/retained"],
17031        ] {
17032            git::git(repo, &args).await.unwrap();
17033        }
17034    }
17035
17036    #[cfg(unix)]
17037    fn fresh_base_interval_fixture(runner: &mut Runner) {
17038        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17039        runner.state.base_sync.as_mut().unwrap().conflict = None;
17040        runner.state.base_sync_recovery = Default::default();
17041        runner.state.status = RunStatus::Voting;
17042    }
17043
17044    #[cfg(unix)]
17045    #[tokio::test]
17046    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17047        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17048        fresh_base_interval_fixture(&mut runner);
17049        runner.state.begin_pass();
17050        runner.sync_to_base().await.unwrap();
17051        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17052        runner.state.resource_block = Some(crate::run::ResourceBlock {
17053            node: "gate".into(),
17054            kind: "build cache".into(),
17055            detail: "busy".into(),
17056            at: Timestamp::now(),
17057            cause: crate::run::ResourceBlockCause::Timeout {
17058                waited_secs: 1,
17059                limit_secs: 1,
17060            },
17061        });
17062        runner.state.save().unwrap();
17063        let queue = crate::queue::Queue::open();
17064        let mut owner = queue.get(&task).unwrap();
17065        runner.state.task_instruction = Some(owner.instruction.clone());
17066        owner.release_operator("fixture-owner", None);
17067        queue.put(&mut owner).unwrap();
17068        let approval = owner.resumption.clone().unwrap();
17069        runner.state.resource_block = None;
17070        runner.state.save().unwrap();
17071        runner.state = RunState::load(&runner.state.id).unwrap();
17072        advance_base_fixture(&runner).await;
17073        runner.sync_to_base().await.unwrap();
17074        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17075        assert_eq!(runner.state.base_episode_used(), 1);
17076        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17077        assert!(
17078            runner.state.base_sync_recovery.episodes[0]
17079                .boundary
17080                .as_ref()
17081                .unwrap()
17082                .consumed
17083        );
17084        assert!(
17085            !runner
17086                .state
17087                .renew_base_episode(&task, &approval, &head, "base")
17088        );
17089        for _ in 0..3 {
17090            owner.release_operator("fixture-owner", None);
17091            queue.put(&mut owner).unwrap();
17092            runner.state = RunState::load(&runner.state.id).unwrap();
17093            runner.state.begin_pass();
17094            runner.sync_to_base().await.unwrap();
17095            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17096            assert_eq!(runner.state.base_episode_used(), 1);
17097        }
17098        runner.state.resource_block = Some(crate::run::ResourceBlock {
17099            node: "gate".into(),
17100            kind: "disk pressure".into(),
17101            detail: "free space required".into(),
17102            at: Timestamp::now(),
17103            cause: crate::run::ResourceBlockCause::Other,
17104        });
17105        runner.state.end_pass_parked(ParkCause::Interrupt {
17106            reason: "disk pressure".into(),
17107        });
17108        runner.state.save().unwrap();
17109        owner.release_operator("fixture-owner-after-disk", None);
17110        queue.put(&mut owner).unwrap();
17111        runner.state.resource_block = None;
17112        advance_base_fixture(&runner).await;
17113        runner.sync_to_base().await.unwrap();
17114        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17115        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17116        assert_eq!(runner.state.base_episode_used(), 1);
17117        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17118    }
17119
17120    #[cfg(unix)]
17121    #[tokio::test]
17122    async fn base_recovery_continuous_advancement_stops_at_four() {
17123        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17124        fresh_base_interval_fixture(&mut runner);
17125        for spent in 1..=BASE_SYNC_ROUNDS {
17126            runner.sync_to_base().await.unwrap();
17127            assert!(!runner.state.base_sync_blocked());
17128            assert_eq!(runner.state.base_episode_used(), spent);
17129            advance_base_fixture(&runner).await;
17130        }
17131        runner.sync_to_base().await.unwrap();
17132        assert_eq!(
17133            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17134            crate::base_sync_recovery::StopKind::Exhausted
17135        );
17136        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17137        assert!(stop.unverified_sync);
17138        let action = runner.state.base_sync_next_action();
17139        assert!(action.contains("never reached its gate"));
17140        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17141        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17142        runner.state = RunState::load(&runner.state.id).unwrap();
17143        runner.sync_to_base().await.unwrap();
17144        assert_eq!(runner.state.base_episode_used(), 4);
17145        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17146    }
17147
17148    #[cfg(unix)]
17149    #[tokio::test]
17150    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17151        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17152        runner.state.ensure_base_episode();
17153        // Verification progress alone is not evidence of a successful sync.
17154        runner.state.record_base_progress(&head);
17155        runner.state.base_sync_recovery.stop = None;
17156        runner.state.record_base_sync_stop(&head);
17157        assert!(
17158            !runner
17159                .state
17160                .base_sync_recovery
17161                .stop
17162                .as_ref()
17163                .unwrap()
17164                .unverified_sync
17165        );
17166
17167        runner.state.finish_base_operation(Some(&head));
17168        // A gate for another head cannot verify this synchronization.
17169        runner.state.gate_ran = true;
17170        runner.state.gate_head = Some("older-head".into());
17171        runner.state.base_sync_recovery.stop = None;
17172        runner.state.record_base_sync_stop(&head);
17173        runner.state.save().unwrap();
17174        runner.state = RunState::load(&runner.state.id).unwrap();
17175        assert!(
17176            runner
17177                .state
17178                .base_sync_recovery
17179                .stop
17180                .as_ref()
17181                .unwrap()
17182                .unverified_sync
17183        );
17184
17185        runner.state.gate_head = Some(head.clone());
17186        runner.state.base_sync_recovery.stop = None;
17187        runner.state.record_base_sync_stop(&head);
17188        assert!(
17189            !runner
17190                .state
17191                .base_sync_recovery
17192                .stop
17193                .as_ref()
17194                .unwrap()
17195                .unverified_sync
17196        );
17197        assert!(
17198            !runner
17199                .state
17200                .base_sync_next_action()
17201                .contains("never reached its gate")
17202        );
17203
17204        // Historical records cannot establish the missing synchronization evidence.
17205        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17206        legacy["base_sync_recovery"]["stop"]
17207            .as_object_mut()
17208            .unwrap()
17209            .remove("unverified_sync");
17210        legacy["base_sync_recovery"]["episodes"][0]
17211            .as_object_mut()
17212            .unwrap()
17213            .remove("last_sync_head");
17214        runner.state = serde_json::from_value(legacy).unwrap();
17215        assert!(
17216            !runner
17217                .state
17218                .base_sync_recovery
17219                .stop
17220                .as_ref()
17221                .unwrap()
17222                .unverified_sync
17223        );
17224        assert!(
17225            runner.state.base_sync_recovery.episodes[0]
17226                .last_sync_head
17227                .is_none()
17228        );
17229    }
17230
17231    #[cfg(unix)]
17232    #[tokio::test]
17233    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17234        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17235        runner.state.authorize_base_sync(&task).unwrap();
17236        assert!(
17237            runner
17238                .state
17239                .base_sync_next_action()
17240                .contains("Approval recorded")
17241        );
17242        assert!(
17243            runner
17244                .state
17245                .base_sync_next_action()
17246                .contains("4/4 in episode 0, 4 lifetime")
17247        );
17248        assert!(runner.state.consume_base_recovery(&head));
17249        assert!(
17250            runner
17251                .state
17252                .base_sync_next_action()
17253                .contains("0/4 in episode 1, 4 lifetime")
17254        );
17255    }
17256
17257    #[cfg(unix)]
17258    #[tokio::test]
17259    async fn base_recovery_changed_head_invalidates_without_spending() {
17260        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17261        runner.state.authorize_base_sync(&task).unwrap();
17262        git::git(
17263            &runner.state.repo,
17264            &["commit", "--allow-empty", "-m", "external head change"],
17265        )
17266        .await
17267        .unwrap();
17268        runner.sync_to_base().await.unwrap();
17269        let a = runner
17270            .state
17271            .base_sync_recovery
17272            .authorizations
17273            .last()
17274            .unwrap();
17275        assert!(a.consumed);
17276        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17277        runner.state = RunState::load(&runner.state.id).unwrap();
17278        assert!(
17279            runner
17280                .state
17281                .base_sync_next_action()
17282                .contains("Approval invalidated")
17283        );
17284        assert!(!runner.state.base_recovery_pending());
17285        assert!(runner.state.base_sync_blocked());
17286        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17287    }
17288
17289    #[cfg(unix)]
17290    #[tokio::test]
17291    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17292        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17293        runner.state.driver_exited = true;
17294        runner.state.save().unwrap();
17295        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17296            .await
17297            .unwrap();
17298        runner.state = RunState::load(&runner.state.id).unwrap();
17299        advance_base_fixture(&runner).await;
17300        runner.sync_to_base().await.unwrap();
17301        let a = runner
17302            .state
17303            .base_sync_recovery
17304            .authorizations
17305            .last()
17306            .unwrap();
17307        assert!(
17308            a.consumed
17309                && a.invalidated
17310                    .as_deref()
17311                    .unwrap()
17312                    .contains("fetched base changed")
17313        );
17314        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17315        assert_eq!(
17316            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17317            head
17318        );
17319        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17320            .await
17321            .unwrap();
17322        runner.state = RunState::load(&runner.state.id).unwrap();
17323        runner.sync_to_base().await.unwrap();
17324        assert!(!runner.state.base_sync_blocked());
17325        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17326    }
17327
17328    #[cfg(unix)]
17329    #[tokio::test]
17330    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17331        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17332        runner.state.authorize_base_sync(&task).unwrap();
17333        assert!(runner.state.consume_base_recovery(&head));
17334        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17335        runner.state.reserve_base_operation(&head, &tip);
17336        runner.state.save().unwrap();
17337        let scratch = runner.state.dir().join("base-sync");
17338        assert_eq!(
17339            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17340                .await
17341                .unwrap(),
17342            crate::rebase::Rebased::Applied
17343        );
17344        runner.state = RunState::load(&runner.state.id).unwrap();
17345        runner.sync_to_base().await.unwrap();
17346        assert!(
17347            runner
17348                .state
17349                .base_sync_recovery
17350                .operations
17351                .last()
17352                .unwrap()
17353                .finished
17354        );
17355        assert_eq!(runner.state.base_episode_used(), 1);
17356        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17357        assert!(!runner.state.base_sync_blocked());
17358    }
17359
17360    #[cfg(unix)]
17361    #[tokio::test]
17362    async fn base_recovery_crash_preserves_untracked_files() {
17363        for ignored in [false, true] {
17364            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17365            runner.state.authorize_base_sync(&task).unwrap();
17366            assert!(runner.state.consume_base_recovery(&head));
17367            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17368            runner.state.reserve_base_operation(&head, &tip);
17369            runner.state.save().unwrap();
17370            let scratch = runner.state.dir().join("base-sync");
17371            assert_eq!(
17372                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17373                    .await
17374                    .unwrap(),
17375                crate::rebase::Rebased::Applied
17376            );
17377            let repo = &runner.state.repo;
17378            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17379            assert_ne!(result, head);
17380            if ignored {
17381                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17382                    .await
17383                    .unwrap();
17384                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17385            }
17386            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17387            assert!(
17388                git::git_raw(repo, &["diff", "--quiet", &head])
17389                    .await
17390                    .unwrap()
17391                    .ok()
17392            );
17393            // Resume after the branch moved, before its checkout was reset.
17394            runner.state = RunState::load(&runner.state.id).unwrap();
17395            runner.sync_to_base().await.unwrap();
17396            assert!(runner.state.base_sync_blocked());
17397            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17398            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17399            assert_eq!(
17400                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17401                result
17402            );
17403            assert_eq!(
17404                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17405                "retained work\n"
17406            );
17407            assert!(
17408                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17409                    .await
17410                    .unwrap()
17411                    .ok()
17412            );
17413        }
17414    }
17415
17416    #[cfg(unix)]
17417    #[tokio::test]
17418    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17419        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17420        runner.state.authorize_base_sync(&task).unwrap();
17421        assert!(runner.state.consume_base_recovery(&head));
17422        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17423        runner.state.reserve_base_operation(&head, &tip);
17424        runner.state.save().unwrap();
17425        let scratch = runner.state.dir().join("base-sync");
17426        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17427            .await
17428            .unwrap();
17429        git::sync_to_head(&runner.state.repo).await.unwrap();
17430        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17431        git::git(&runner.state.repo, &["add", "foreign"])
17432            .await
17433            .unwrap();
17434        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17435            .await
17436            .unwrap();
17437        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17438        runner.state = RunState::load(&runner.state.id).unwrap();
17439        runner.sync_to_base().await.unwrap();
17440        assert!(runner.state.base_sync_blocked());
17441        assert!(
17442            !runner
17443                .state
17444                .base_sync_recovery
17445                .operations
17446                .last()
17447                .unwrap()
17448                .finished
17449        );
17450        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17451        assert_eq!(
17452            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17453            foreign_head
17454        );
17455    }
17456
17457    #[cfg(unix)]
17458    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17459    {
17460        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17461        fresh_base_interval_fixture(&mut runner);
17462        let repo = &runner.state.repo;
17463        git::git(repo, &["checkout", "main"]).await.unwrap();
17464        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17465        git::git(repo, &["add", "retained"]).await.unwrap();
17466        git::git(repo, &["commit", "-m", "upstream retained change"])
17467            .await
17468            .unwrap();
17469        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17470        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17471        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17472        let scratch = runner.state.dir().join("base-sync");
17473        assert!(matches!(
17474            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17475                .await
17476                .unwrap(),
17477            git::RebaseStart::Conflicted(_)
17478        ));
17479        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17480            agent: "fixture".into(),
17481            paths: vec!["retained".into()],
17482            from: Some(head.clone()),
17483            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17484            index_outside: None,
17485            finished: false,
17486            error: None,
17487            launcher: None,
17488            infra: None,
17489            outage: None,
17490        });
17491        (tmp, runner, head, tip, scratch)
17492    }
17493
17494    #[cfg(unix)]
17495    #[tokio::test]
17496    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17497        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17498        // A resource stop from before reservations existed, with resolved
17499        // contents left for the launcher's ordinary staging continuation.
17500        std::fs::write(
17501            scratch.join("retained"),
17502            "upstream and retained resolution\n",
17503        )
17504        .unwrap();
17505        runner.state.resource_block = Some(crate::run::ResourceBlock {
17506            node: "rebase".into(),
17507            kind: "repository coordination".into(),
17508            detail: "busy".into(),
17509            at: Timestamp::now(),
17510            cause: crate::run::ResourceBlockCause::Other,
17511        });
17512        runner.state.save().unwrap();
17513        // Advance the remote without moving either checkout participating
17514        // in the standing rebase.
17515        let newer = git::git(
17516            &runner.state.repo,
17517            &[
17518                "commit-tree",
17519                "main^{tree}",
17520                "-p",
17521                "main",
17522                "-m",
17523                "base advanced during wait",
17524            ],
17525        )
17526        .await
17527        .unwrap();
17528        git::git(
17529            &runner.state.repo,
17530            &[
17531                "push",
17532                "origin",
17533                &format!("{}:refs/heads/main", newer.trim()),
17534            ],
17535        )
17536        .await
17537        .unwrap();
17538        runner.state = RunState::load(&runner.state.id).unwrap();
17539        runner.state.resource_block = None;
17540        runner.sync_to_base().await.unwrap();
17541        assert!(!runner.state.base_sync_blocked());
17542        let op = &runner.state.base_sync_recovery.operations[0];
17543        assert_eq!(op.head, head);
17544        assert_eq!(op.tip, tip);
17545        assert!(op.finished);
17546        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17547        assert_eq!(runner.state.rebase_fixes.len(), 1);
17548        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17549            .await
17550            .unwrap();
17551        assert_ne!(latest, tip);
17552        runner.state = RunState::load(&runner.state.id).unwrap();
17553        runner.sync_to_base().await.unwrap();
17554        assert!(!runner.state.base_sync_blocked());
17555        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17556        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17557        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17558    }
17559
17560    #[cfg(unix)]
17561    #[tokio::test]
17562    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17563        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17564        runner.state.reserve_base_operation(&head, &tip);
17565        runner.state.save().unwrap();
17566        std::fs::write(
17567            scratch.join("retained"),
17568            "upstream and retained resolution\n",
17569        )
17570        .unwrap();
17571        git::git(&scratch, &["add", "retained"]).await.unwrap();
17572        git::git(
17573            &scratch,
17574            &["-c", "core.editor=true", "rebase", "--continue"],
17575        )
17576        .await
17577        .unwrap();
17578        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17579        assert!(
17580            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17581                .await
17582                .unwrap()
17583                .is_empty()
17584        );
17585        assert_eq!(
17586            crate::rebase::finish(
17587                &mut runner.state,
17588                &scratch,
17589                "fix/retained",
17590                &head,
17591                &tip,
17592                &["retained".into()],
17593                ""
17594            )
17595            .await
17596            .unwrap(),
17597            crate::rebase::Rebased::Applied
17598        );
17599        assert!(!scratch.exists());
17600        // Simulate losing the caller before finish_base_operation/save.
17601        runner.state = RunState::load(&runner.state.id).unwrap();
17602        let op = &runner.state.base_sync_recovery.operations[0];
17603        assert!(!op.finished);
17604        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17605        runner.sync_to_base().await.unwrap();
17606        assert!(!runner.state.base_sync_blocked());
17607        assert!(runner.state.base_sync_recovery.operations[0].finished);
17608        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17609        assert_eq!(runner.state.rebase_fixes.len(), 1);
17610        let winner = runner.state.winner().unwrap().clone();
17611        let wait = runner.repository_wait();
17612        assert!(
17613            verify_current_head(&mut runner.state, &winner, &wait)
17614                .await
17615                .unwrap()
17616        );
17617        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17618    }
17619
17620    #[cfg(unix)]
17621    #[tokio::test]
17622    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17623        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17624            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17625            runner.state.reserve_base_operation(&head, &tip);
17626            runner.state.save().unwrap();
17627            std::fs::write(
17628                scratch.join("retained"),
17629                "upstream and retained resolution\n",
17630            )
17631            .unwrap();
17632            git::git(&scratch, &["add", "retained"]).await.unwrap();
17633            git::git(
17634                &scratch,
17635                &["-c", "core.editor=true", "rebase", "--continue"],
17636            )
17637            .await
17638            .unwrap();
17639            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17640            assert!(!git::rebase_in_progress(&scratch).await);
17641            assert!(
17642                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17643                    .await
17644                    .unwrap()
17645                    .is_empty()
17646            );
17647            match damage {
17648                Some("record") => {
17649                    let path =
17650                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17651                            .await
17652                            .unwrap();
17653                    std::fs::write(scratch.join(path), "").unwrap();
17654                }
17655                Some("foreign") => {
17656                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17657                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17658                    git::git(&scratch, &["commit", "-m", "foreign work"])
17659                        .await
17660                        .unwrap();
17661                }
17662                Some("dirty") => {
17663                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17664                }
17665                _ => {}
17666            }
17667            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17668            // Crash immediately after Git continuation, before finish runs.
17669            runner.state = RunState::load(&runner.state.id).unwrap();
17670            assert!(
17671                runner.state.base_sync_recovery.operations[0]
17672                    .validated_head
17673                    .is_none()
17674            );
17675            runner.sync_to_base().await.unwrap();
17676            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17677            assert_eq!(runner.state.base_episode_used(), 1);
17678            assert_eq!(runner.state.rebase_fixes.len(), 1);
17679            assert_eq!(
17680                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17681                retained_head
17682            );
17683            if damage.is_some() {
17684                assert!(runner.state.base_sync_blocked());
17685                assert!(scratch.exists());
17686                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17687                assert!(
17688                    runner.state.base_sync_recovery.operations[0]
17689                        .validated_head
17690                        .is_none()
17691                );
17692            } else {
17693                assert!(!runner.state.base_sync_blocked());
17694                assert!(!scratch.exists());
17695                assert!(runner.state.base_sync_recovery.operations[0].finished);
17696                assert_eq!(
17697                    runner.state.base_sync_recovery.operations[0]
17698                        .validated_head
17699                        .as_deref(),
17700                    Some(result.as_str())
17701                );
17702                let winner = runner.state.winner().unwrap().clone();
17703                let wait = runner.repository_wait();
17704                assert!(
17705                    verify_current_head(&mut runner.state, &winner, &wait)
17706                        .await
17707                        .unwrap()
17708                );
17709                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17710                runner.state = RunState::load(&runner.state.id).unwrap();
17711                runner.sync_to_base().await.unwrap();
17712                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17713            }
17714        }
17715    }
17716
17717    #[cfg(unix)]
17718    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17719        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17720        // This is an ordinary gate stop, not an authorized base-budget recovery.
17721        runner.state.prep_complete = Some(true);
17722        runner.state.base_sync = None;
17723        runner.state.base_sync_recovery = Default::default();
17724        runner.state.config.graph.gate_fix_rounds = 1;
17725        let mut review = review_round(true, 0, 1, 1, false, true);
17726        review.head = head.clone();
17727        review.verified_head = Some(head.clone());
17728        runner.state.reviews = vec![review];
17729        runner.state.gate = vec![CommandOutcome {
17730            command: "old gate".into(),
17731            code: Some(1),
17732            output_tail: "old failure".into(),
17733            duration_ms: 0,
17734            resource_blocked: false,
17735        }];
17736        runner.state.gate_head = Some(head.clone());
17737        runner.state.gate_ran = true;
17738        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17739            agent: "fixture".into(),
17740            failed: runner.state.gate.clone(),
17741            notes: "spent".into(),
17742            committed: false,
17743            error: None,
17744        }];
17745        runner.state.land_approval = Some(crate::run::LandApproval {
17746            question: "old approval".into(),
17747            head: head.clone(),
17748        });
17749        runner.state.status = RunStatus::Blocked;
17750        (tmp, runner, head)
17751    }
17752
17753    #[cfg(unix)]
17754    #[tokio::test]
17755    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17756        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17757            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17758            let phases = tmp.path().join("phases");
17759            runner.state.config.verify.e2e = vec![format!(
17760                "echo e2e >> '{}'; {}",
17761                phases.display(),
17762                if e2e_ok {
17763                    "true"
17764                } else {
17765                    "echo new-e2e-failure; false"
17766                }
17767            )];
17768            runner.state.config.verify.gate = vec![format!(
17769                "echo gate >> '{}'; {}",
17770                phases.display(),
17771                if gate_ok {
17772                    "true"
17773                } else {
17774                    "echo new-gate-failure; false"
17775                }
17776            )];
17777            runner.state.save().unwrap();
17778            runner.state = RunState::load(&runner.state.id).unwrap();
17779            runner.sync_to_base().await.unwrap();
17780            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17781            assert_ne!(head, old);
17782            // Reentry after Git updated but before verification was invalidated.
17783            runner.state = RunState::load(&runner.state.id).unwrap();
17784            runner.execute_graph().await.unwrap();
17785            assert_eq!(runner.state.gate_fixes.len(), 1);
17786            assert_eq!(runner.state.reviews.len(), 1);
17787            assert_eq!(runner.state.reviews[0].head, old);
17788            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17789            assert_eq!(runner.state.verification_history.len(), 1);
17790            let history = &runner.state.verification_history[0];
17791            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17792            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17793            assert!(history.archived_at.is_some());
17794            assert_eq!(history.fixes_spent, Some(1));
17795            assert_eq!(history.gate[0].output_tail, "old failure");
17796            let archive_events: Vec<_> = runner
17797                .state
17798                .events
17799                .iter()
17800                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17801                .collect();
17802            assert_eq!(archive_events.len(), 1);
17803            assert!(archive_events[0].message.contains(&short(&old)));
17804            assert!(archive_events[0].message.contains(&short(&head)));
17805            assert!(
17806                archive_events[0]
17807                    .message
17808                    .contains("1 gate-fix round(s) already spent stay spent")
17809            );
17810            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17811            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17812            if e2e_ok {
17813                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17814                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17815            } else {
17816                assert!(!runner.state.gate_ran);
17817            }
17818            assert_eq!(
17819                runner.state.status,
17820                if e2e_ok && gate_ok {
17821                    RunStatus::Ready
17822                } else {
17823                    RunStatus::Blocked
17824                }
17825            );
17826            if !e2e_ok || !gate_ok {
17827                let cause = runner.state.verification_blocking_cause().unwrap();
17828                assert!(cause.contains(&head));
17829                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17830                // A later successful progress event must not become the cause.
17831                runner.state.event("land", "rebased successfully");
17832                let notice = crate::notices::run_ended(&runner.state).unwrap();
17833                assert!(!format!("{notice:?}").contains("rebased successfully"));
17834                assert!(runner.state.merge.is_none());
17835            }
17836            let before = std::fs::read_to_string(&phases).unwrap();
17837            runner.state.save().unwrap();
17838            runner.state = RunState::load(&runner.state.id).unwrap();
17839            runner.review_loop().await.unwrap();
17840            runner.gate().await.unwrap();
17841            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17842            assert_eq!(runner.state.verification_history.len(), 1);
17843            let history = &runner.state.verification_history[0];
17844            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17845            assert!(history.archived_at.is_some());
17846            assert_eq!(history.fixes_spent, Some(1));
17847        }
17848    }
17849
17850    #[cfg(unix)]
17851    #[tokio::test]
17852    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17853        let (tmp, mut runner, head) = resumed_gate_fixture().await;
17854        let marker = tmp.path().join("gate-ran");
17855        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17856        runner.review_loop().await.unwrap();
17857        runner.gate().await.unwrap();
17858        assert_eq!(runner.state.status, RunStatus::Blocked);
17859        assert!(!marker.exists());
17860        assert!(runner.state.verification_history.is_empty());
17861        runner.state.gate_head = None;
17862        runner.gate().await.unwrap();
17863        assert!(marker.exists());
17864        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17865        assert_eq!(runner.state.verification_history.len(), 1);
17866        assert_eq!(runner.state.gate_fixes.len(), 1);
17867    }
17868
17869    #[cfg(unix)]
17870    #[tokio::test]
17871    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17872        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17873        runner.sync_to_base().await.unwrap();
17874        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17875        runner.state.reconcile_verification_head(&head);
17876        // Both crash-before-command and resource wait are unresolved evidence.
17877        for outcomes in [
17878            Vec::new(),
17879            vec![CommandOutcome {
17880                command: "true".into(),
17881                code: None,
17882                output_tail: "cache unavailable".into(),
17883                duration_ms: 0,
17884                resource_blocked: true,
17885            }],
17886        ] {
17887            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17888                head: head.clone(),
17889                passed: false,
17890                outcomes,
17891            });
17892            runner.state.gate_ran = false;
17893            runner.state.gate.clear();
17894            runner.state.save().unwrap();
17895            runner.state = RunState::load(&runner.state.id).unwrap();
17896            runner.merge().await.unwrap();
17897            assert!(runner.state.merge.is_none());
17898            runner.review_loop().await.unwrap();
17899            runner.gate().await.unwrap();
17900            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17901            assert!(runner.state.gate_allows_progress());
17902            assert_eq!(runner.state.gate_fixes.len(), 1);
17903            assert!(runner.state.merge.is_none());
17904        }
17905        // Moving the winner after a successful gate cannot publish an unchecked SHA.
17906        git::git(
17907            &runner.state.repo,
17908            &["commit", "--allow-empty", "-m", "external update"],
17909        )
17910        .await
17911        .unwrap();
17912        runner.merge().await.unwrap();
17913        assert!(runner.state.merge.is_none());
17914        assert_eq!(runner.state.status, RunStatus::Blocked);
17915    }
17916
17917    #[cfg(unix)]
17918    #[tokio::test]
17919    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17920        for waiting_on_gate in [false, true] {
17921            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17922            runner.sync_to_base().await.unwrap();
17923            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17924            runner.state.reconcile_verification_head(&head);
17925            if waiting_on_gate {
17926                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17927                    head: head.clone(),
17928                    passed: true,
17929                    outcomes: Vec::new(),
17930                });
17931            }
17932            let cache = tmp.path().join("cache");
17933            runner.state.config.verify.e2e =
17934                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17935            runner.state.config.verify.gate = vec!["true".into()];
17936            runner.state.config.graph.timeout_resource_wait = 1;
17937            let owner =
17938                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17939            let guard =
17940                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17941                    crate::cache::AcquireOutcome::Acquired(g) => g,
17942                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17943                };
17944            runner.review_loop().await.unwrap();
17945            runner.gate().await.unwrap();
17946            assert!(!runner.state.gate_ran);
17947            assert_eq!(runner.state.gate_fixes.len(), 1);
17948            assert!(runner.state.resource_block.is_some());
17949            runner.merge().await.unwrap();
17950            assert!(runner.state.merge.is_none());
17951            drop(guard);
17952            runner.state = RunState::load(&runner.state.id).unwrap();
17953            // The driver clears the previous pass's resource stop on resume.
17954            runner.state.resource_block = None;
17955            runner.review_loop().await.unwrap();
17956            runner.gate().await.unwrap();
17957            assert!(runner.state.gate_allows_progress());
17958            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17959            assert_eq!(runner.state.gate_fixes.len(), 1);
17960        }
17961    }
17962
17963    #[cfg(unix)]
17964    #[tokio::test]
17965    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
17966        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17967        runner.sync_to_base().await.unwrap();
17968        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17969        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
17970        // Verification passes before pre_gate and fails on the mechanically changed head.
17971        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
17972        runner.review_loop().await.unwrap();
17973        runner.gate().await.unwrap();
17974        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17975        assert_ne!(before, after);
17976        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
17977        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
17978        assert!(!runner.state.gate_allows_progress());
17979        runner.merge().await.unwrap();
17980        assert!(runner.state.merge.is_none());
17981        assert!(
17982            runner
17983                .state
17984                .verification_history
17985                .iter()
17986                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
17987        );
17988
17989        // An incomplete review is still an independent hold on a changed head.
17990        runner.state.reviews[0].clean = false;
17991        runner.state.reviews[0].answered = 0;
17992        runner.state.config.verify.e2e = vec!["true".into()];
17993        runner.review_loop().await.unwrap();
17994        runner.gate().await.unwrap();
17995        runner.merge().await.unwrap();
17996        assert_eq!(runner.state.status, RunStatus::Blocked);
17997        assert!(runner.state.merge.is_none());
17998        assert!(
17999            runner
18000                .state
18001                .verification_blocking_cause()
18002                .unwrap()
18003                .contains("review blocked")
18004        );
18005    }
18006
18007    #[cfg(unix)]
18008    #[tokio::test]
18009    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18010        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18011        runner.state.gate_head = None;
18012        runner.state.config.verify.gate =
18013            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18014        runner.review_loop().await.unwrap();
18015        runner.gate().await.unwrap();
18016        assert_ne!(
18017            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18018            old
18019        );
18020        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18021        assert!(!runner.state.gate_allows_progress());
18022        assert_eq!(runner.state.status, RunStatus::Blocked);
18023        runner.merge().await.unwrap();
18024        assert!(runner.state.merge.is_none());
18025    }
18026
18027    #[cfg(unix)]
18028    #[tokio::test]
18029    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18030        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18031        let mut old_review = review_round(true, 0, 1, 1, false, true);
18032        old_review.head = head.clone();
18033        old_review.verified_head = Some(head.clone());
18034        runner.state.reviews = vec![old_review];
18035        runner.state.gate = vec![CommandOutcome {
18036            command: "old gate".into(),
18037            code: None,
18038            output_tail: "timed out after 2400s".into(),
18039            duration_ms: 2_400_000,
18040            resource_blocked: false,
18041        }];
18042        runner.state.gate_ran = true;
18043        runner.state.gate_head = Some(head.clone());
18044        runner.state.land_approval = Some(crate::run::LandApproval {
18045            question: "old-head-approval".into(),
18046            head: head.clone(),
18047        });
18048        runner.state.authorize_base_sync(&task).unwrap();
18049        runner.sync_to_base().await.unwrap();
18050        assert!(!runner.state.base_sync_blocked());
18051        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18052        assert_ne!(head, fresh);
18053        assert_eq!(runner.state.reviews.len(), 1);
18054        assert!(!runner.state.gate_ran);
18055        let saved = &runner.state.base_sync_recovery.authorizations[0];
18056        assert_eq!(saved.reviews[0].head, head);
18057        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18058        assert!(saved.gate_ran);
18059        assert!(!saved.gate[0].ok());
18060        runner.review_loop().await.unwrap();
18061        runner.gate().await.unwrap();
18062        assert_eq!(runner.state.reviews[0].head, head);
18063        assert_eq!(
18064            runner.state.reviews[0].verified_head.as_deref(),
18065            Some(head.as_str())
18066        );
18067        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18068        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18069        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18070        assert!(runner.state.gate_ran);
18071        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18072        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18073        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18074        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18075        assert_eq!(
18076            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18077            "retained change"
18078        );
18079    }
18080
18081    #[cfg(unix)]
18082    #[tokio::test]
18083    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18084        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18085        let queue = crate::queue::Queue::open();
18086        let mut owner = queue.get(&task).unwrap();
18087        owner.release_operator("test-resume", None);
18088        queue.put(&mut owner).unwrap();
18089        let generation = owner.resumption.as_ref().unwrap().generation;
18090        runner.state.task_instruction = Some(owner.instruction.clone());
18091        runner.state.driver_exited = true;
18092        runner.state.save().unwrap();
18093        let home = crate::run::home();
18094        let _claim = queue.claim(&task).unwrap();
18095        owner.resumption.as_mut().unwrap().at =
18096            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18097        queue.put(&mut owner).unwrap();
18098        assert!(
18099            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18100                .await
18101                .is_err(),
18102            "an approval from before the stop grants nothing"
18103        );
18104        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18105        queue.put(&mut owner).unwrap();
18106        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18107            .await
18108            .unwrap();
18109        assert!(
18110            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18111                .await
18112                .is_err()
18113        );
18114        runner.state = RunState::load(&runner.state.id).unwrap();
18115        assert_eq!(
18116            runner.state.base_sync_recovery.authorizations[0].generation,
18117            Some(generation)
18118        );
18119        runner.sync_to_base().await.unwrap();
18120        let repo = runner.state.repo.clone();
18121        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18122        git::git(&repo, &["checkout", "main"]).await.unwrap();
18123        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18124        git::git(&repo, &["add", "."]).await.unwrap();
18125        git::git(&repo, &["commit", "-m", "move base again"])
18126            .await
18127            .unwrap();
18128        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18129        git::git(&repo, &["checkout", "fix/retained"])
18130            .await
18131            .unwrap();
18132        runner.sync_to_base().await.unwrap();
18133        assert!(!runner.state.base_sync_blocked());
18134        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18135        assert_eq!(runner.state.base_episode_used(), 2);
18136        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18137        assert_ne!(fresh, head);
18138        assert!(!runner.state.base_recovery_pending());
18139        // Even replaying the generation with a later timestamp grants nothing.
18140        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18141        queue.put(&mut owner).unwrap();
18142        assert!(
18143            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18144                .await
18145                .is_err()
18146        );
18147        runner.state = RunState::load(&runner.state.id).unwrap();
18148        runner.sync_to_base().await.unwrap();
18149        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18150        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18151        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18152    }
18153
18154    #[cfg(unix)]
18155    #[tokio::test]
18156    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18157        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18158        runner.state.base_sync = None;
18159        runner.state.status = RunStatus::Blocked;
18160        let mut second = runner.state.candidates[0].clone();
18161        second.label = 'B';
18162        runner.state.candidates.push(second);
18163        assert!(runner.state.votes.is_empty());
18164        assert!(runner.state.deliberation.is_empty());
18165        runner.deliberate().await.unwrap();
18166        runner.vote().await.unwrap();
18167        assert_eq!(runner.state.status, RunStatus::Blocked);
18168        assert!(runner.state.votes.is_empty());
18169        assert!(runner.state.deliberation.is_empty());
18170    }
18171
18172    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18173    /// reads nothing else off it.
18174    fn runner_at(status: RunStatus) -> Runner {
18175        let mut state = RunState::new(
18176            PathBuf::from("/nonexistent/repo"),
18177            "main".to_owned(),
18178            "deadbeef".to_owned(),
18179            "task".to_owned(),
18180            Config::default(),
18181        );
18182        // These fixtures test execution flow, independent of host disk capacity.
18183        state.config.disk.min_free_bytes = 0;
18184        // These fixtures share one home and save concurrently. Short clock
18185        // IDs can collide, making an ownership test read another test's run.
18186        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18187        state.id = format!(
18188            "20000101-000000-{:04x}",
18189            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18190        );
18191        state.status = status;
18192        Runner {
18193            state,
18194            roles: ResolvedRoles {
18195                implementers: Vec::new(),
18196                judges: Vec::new(),
18197                reviewers: Vec::new(),
18198                fixer: None,
18199                conductor: conductor(),
18200                implementer_roster: Vec::new(),
18201                judge_roster: Vec::new(),
18202                reviewer_roster: Vec::new(),
18203            },
18204            sem: Arc::new(Semaphore::new(1)),
18205            pause: Pause::new(),
18206            interrupt: Pause::new(),
18207        }
18208    }
18209
18210    #[tokio::test]
18211    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18212        crate::run::pin_test_home();
18213        let temp = tempfile::tempdir().unwrap();
18214        let root = temp.path().canonicalize().unwrap();
18215        let repo = root.join("repo");
18216        std::fs::create_dir(&repo).unwrap();
18217        init_repo(&repo);
18218        git::git(&repo, &["config", "core.autocrlf", "false"])
18219            .await
18220            .unwrap();
18221        let mut runner = runner_at(RunStatus::Blocked);
18222        runner.state.repo = repo.clone();
18223        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18224        runner.state.config.agents = vec![conductor()];
18225        runner.state.config.graph.selected_implementer = Some("conductor".into());
18226        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18227        let (shell, command, setup_output) = if cfg!(windows) {
18228            (
18229                vec!["cmd".into(), "/C".into()],
18230                "echo ready>seat-ready",
18231                "ready\r\n",
18232            )
18233        } else {
18234            (
18235                vec!["sh".into(), "-c".into()],
18236                "printf ready > seat-ready",
18237                "ready",
18238            )
18239        };
18240        runner.state.config.verify.shell = Some(shell);
18241        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18242            run: Some(command.into()),
18243            ..Default::default()
18244        }];
18245        runner.state.save().unwrap();
18246        let path = runner.state.worktree_root().join("judge-1");
18247        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18248            .await
18249            .unwrap();
18250        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18251        git::git(&path, &["add", "."]).await.unwrap();
18252        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18253        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18254            .await
18255            .unwrap();
18256        git::git(&path, &["checkout", "--detach", &head])
18257            .await
18258            .unwrap();
18259        let evidence = crate::fold_evidence::snapshot(
18260            &runner.state,
18261            &crate::run::home().canonicalize().unwrap(),
18262            &head,
18263            64 * 1024 * 1024,
18264        )
18265        .await
18266        .unwrap();
18267        let mut contents = BTreeMap::new();
18268        contents.insert(
18269            repo.join("logs/HEAD"),
18270            format!("{head} {head} fixture\n").into_bytes(),
18271        );
18272        let record = serde_json::json!({
18273            "version":1, "run":runner.state.id, "source":path, "head":head,
18274            "evidence":evidence, "admin":{}, "admin_contents":contents,
18275            "history":{head.clone():evidence}, "phase":"evicted"
18276        });
18277        std::fs::write(
18278            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18279            serde_json::to_vec(&record).unwrap(),
18280        )
18281        .unwrap();
18282        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18283        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18284            .await
18285            .unwrap();
18286        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18287        assert!(
18288            !git::git_raw(&repo, &["cat-file", "-e", &head])
18289                .await
18290                .unwrap()
18291                .ok()
18292        );
18293        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18294        let error = resumed.execute().await.unwrap_err();
18295        assert!(
18296            error
18297                .to_string()
18298                .contains("selected implementation is held"),
18299            "{error:#}"
18300        );
18301        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18302        assert_eq!(
18303            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18304            setup_output
18305        );
18306        assert_eq!(
18307            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18308            "unrelated seat"
18309        );
18310        let error = resumed.execute().await.unwrap_err();
18311        assert!(
18312            error
18313                .to_string()
18314                .contains("selected implementation is held"),
18315            "{error:#}"
18316        );
18317        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18318        git::git(&path, &["add", "."]).await.unwrap();
18319        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18320        let next = git::git(
18321            &path,
18322            &[
18323                "commit-tree",
18324                &next_tree,
18325                "-p",
18326                &head,
18327                "-m",
18328                "next review checkpoint",
18329            ],
18330        )
18331        .await
18332        .unwrap();
18333        git::git(&path, &["checkout", "--detach", &next])
18334            .await
18335            .unwrap();
18336        resumed.state.reviews.push(
18337            serde_json::from_value(serde_json::json!({
18338                "round":1, "head":next, "reviews":[]
18339            }))
18340            .unwrap(),
18341        );
18342        resumed.state.save().unwrap();
18343        let error = resumed.execute().await.unwrap_err();
18344        assert!(
18345            error
18346                .to_string()
18347                .contains("selected implementation is held"),
18348            "{error:#}"
18349        );
18350        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18351    }
18352
18353    #[tokio::test]
18354    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18355        crate::run::pin_test_home();
18356        let mut runner = runner_at(RunStatus::Gating);
18357        runner.state.save().unwrap();
18358        let id = runner.state.id.clone();
18359        let passes = runner.state.passes.len();
18360        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18361        let mut claiming = Box::pin(runner.claim_driver());
18362        std::future::poll_fn(|cx| {
18363            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18364            std::task::Poll::Ready(())
18365        })
18366        .await;
18367        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18368        drop(owner);
18369        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18370            .await
18371            .unwrap()
18372            .unwrap();
18373        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18374        assert!(execution.runner.state.resource_block.is_none());
18375        assert_eq!(
18376            execution.runner.state.liveness(false),
18377            crate::run::Liveness::Live
18378        );
18379        drop(execution);
18380        assert!(RunState::load(&id).unwrap().driver_exited);
18381    }
18382
18383    #[tokio::test]
18384    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18385        crate::run::pin_test_home();
18386        let mut runner = runner_at(RunStatus::Gating);
18387        runner.state.save().unwrap();
18388        // run.json can commit, but the subsequent lease metadata write fails.
18389        // Cleanup must run after releasing the short ownership lock.
18390        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18391        assert!(runner.claim_driver().await.is_err());
18392        assert!(runner.state.driver_exited);
18393        assert_eq!(runner.state.driver_pid, None);
18394        let disk = RunState::load(&runner.state.id).unwrap();
18395        assert!(disk.driver_exited);
18396        assert_eq!(disk.driver_pid, None);
18397        assert_eq!(disk.status, RunStatus::Gating);
18398    }
18399
18400    #[tokio::test]
18401    async fn an_early_graph_error_releases_its_driver() {
18402        crate::run::pin_test_home();
18403        let mut runner = runner_at(RunStatus::Blocked);
18404        runner.state.config.graph.selected_implementer = Some("selected".into());
18405        runner.state.save().unwrap();
18406        assert!(runner.execute().await.is_err());
18407        let disk = RunState::load(&runner.state.id).unwrap();
18408        assert!(disk.driver_exited);
18409        assert_eq!(disk.driver_pid, None);
18410        assert_eq!(disk.status, RunStatus::Blocked);
18411    }
18412
18413    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18414        crate::run::MergeRepair {
18415            question: "repair-question".into(),
18416            head: "old-head".into(),
18417            pr: "https://github.com/o/r/pull/42".into(),
18418            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18419            gate: vec![outcome(Some(0), false)],
18420            gate_head: Some("old-head".into()),
18421            gate_fixes: Vec::new(),
18422            pre_gate: Vec::new(),
18423            phase,
18424            failure: None,
18425        }
18426    }
18427
18428    #[tokio::test]
18429    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18430        crate::run::pin_test_home();
18431        let mut runner = runner_at(RunStatus::Landing);
18432        runner.state.merge_repairs.push(merge_repair_request(
18433            crate::run::MergeRepairPhase::Requested,
18434        ));
18435        runner.state.save().unwrap();
18436        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18437        assert!(
18438            runner
18439                .execute()
18440                .await
18441                .unwrap_err()
18442                .to_string()
18443                .contains("already running")
18444        );
18445        assert_eq!(
18446            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18447            crate::run::MergeRepairPhase::Requested
18448        );
18449        drop(claim);
18450        let mut disk = runner.state.clone();
18451        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18452        disk.save().unwrap();
18453        assert!(
18454            runner
18455                .execute()
18456                .await
18457                .unwrap_err()
18458                .to_string()
18459                .contains("decision changed")
18460        );
18461        assert_eq!(
18462            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18463            crate::run::MergeRepairPhase::Complete
18464        );
18465    }
18466
18467    #[test]
18468    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18469        crate::run::pin_test_home();
18470        let mut runner = runner_at(RunStatus::Landing);
18471        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18472        runner.state.reviews = request.reviews.clone();
18473        runner.state.gate = request.gate.clone();
18474        runner.state.gate_ran = true;
18475        runner.state.land_approval = Some(crate::run::LandApproval {
18476            question: "old".into(),
18477            head: "old-head".into(),
18478        });
18479        runner.state.merge_repairs.push(request);
18480        runner.begin_merge_repair_review(0).unwrap();
18481        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18482        assert!(!runner.state.gate_status().ok());
18483        assert!(runner.state.land_approval.is_none());
18484        let disk = RunState::load(&runner.state.id).unwrap();
18485        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18486        assert!(disk.merge_repairs[0].gate[0].ok());
18487        assert_eq!(
18488            disk.merge_repairs[0].phase,
18489            crate::run::MergeRepairPhase::Reviewing
18490        );
18491        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18492        assert!(disk.reviews.is_empty());
18493    }
18494
18495    #[tokio::test]
18496    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18497        crate::run::pin_test_home();
18498        for phase in [
18499            crate::run::MergeRepairPhase::Fixing,
18500            crate::run::MergeRepairPhase::Requested,
18501        ] {
18502            let mut runner = runner_at(RunStatus::Landing);
18503            runner.state.config.graph.land_rounds = 0;
18504            runner.state.merge_repairs.push(merge_repair_request(phase));
18505            runner.state.save().unwrap();
18506            assert!(!runner.repair_merge_review().await.unwrap());
18507            assert_eq!(runner.state.status, RunStatus::Blocked);
18508            assert_eq!(
18509                runner.state.merge_repairs[0].phase,
18510                crate::run::MergeRepairPhase::Failed
18511            );
18512            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18513            assert!(
18514                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18515                    "interrupted"
18516                } else {
18517                    "budget exhausted"
18518                })
18519            );
18520            assert!(runner.state.merge.is_none());
18521        }
18522    }
18523
18524    /// `park_here` folding in the reason `Pause::park_because` recorded -
18525    /// this is what lets an operator reading a run's events tell an
18526    /// interrupt-driven park from an ordinary shutdown park.
18527    #[test]
18528    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18529        crate::run::pin_test_home();
18530        let mut runner = runner_at(RunStatus::Implementing);
18531        let interrupt = Pause::new();
18532        runner.watch_interrupt(interrupt.clone());
18533
18534        interrupt.park_because("task a1b2 asked to run first");
18535
18536        assert!(runner.park_here().expect("park_here"));
18537        assert!(runner.state.parked);
18538        let last = runner.state.events.last().expect("a park event");
18539        assert_eq!(last.node, "park");
18540        assert!(
18541            last.message.contains("task a1b2 asked to run first"),
18542            "expected the interrupt reason in {:?}",
18543            last.message
18544        );
18545    }
18546
18547    /// The cause is stored on the pass when it parks, and the first cause a
18548    /// shared pause was given is the one that stays.
18549    #[test]
18550    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18551        crate::run::pin_test_home();
18552        let mut runner = runner_at(RunStatus::Implementing);
18553        runner.state.begin_pass();
18554        let shutdown = Pause::new();
18555        runner.on_pause(shutdown.clone());
18556        let upgrade = ParkCause::Upgrade {
18557            from: "0.121.0".to_owned(),
18558            to: Some("0.121.1".to_owned()),
18559        };
18560        shutdown.park_for(upgrade.clone());
18561        shutdown.park_for(ParkCause::Operator);
18562
18563        assert!(runner.park_here().expect("park_here"));
18564        assert_eq!(
18565            runner.state.passes[0].end,
18566            Some(crate::run::PassEnd::Parked { cause: upgrade })
18567        );
18568        let last = runner.state.events.last().expect("a park event");
18569        assert!(last.message.contains("0.121.0"), "{}", last.message);
18570    }
18571
18572    #[tokio::test]
18573    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18574        crate::run::pin_test_home();
18575        let mut runner = runner_at(RunStatus::Reviewing);
18576        runner.state.save().unwrap();
18577        let mut execution = runner.claim_driver().await.expect("first execution");
18578        assert_eq!(execution.runner.state.passes.len(), 1);
18579        execution
18580            .driver
18581            .finish(&mut execution.runner.state)
18582            .await
18583            .unwrap();
18584    }
18585
18586    #[tokio::test]
18587    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18588        crate::run::pin_test_home();
18589        let mut runner = runner_at(RunStatus::Reviewing);
18590        runner.state.config.agents = vec![conductor()];
18591        runner.state.config.graph.timeout_resource_wait = 0;
18592        runner.state.save().unwrap();
18593        let id = runner.state.id.clone();
18594        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18595        assert!(runner.claim_driver().await.is_err());
18596        drop(owner);
18597        drop(runner);
18598        let mut resumed = Runner::resume(&id).unwrap();
18599        assert_eq!(resumed.state.driver_claimed, Some(false));
18600        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18601        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18602        let mut execution = resumed
18603            .claim_driver()
18604            .await
18605            .expect("retry initial claim after reload");
18606        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18607        assert_eq!(execution.runner.state.passes.len(), 1);
18608        execution
18609            .driver
18610            .finish(&mut execution.runner.state)
18611            .await
18612            .unwrap();
18613    }
18614
18615    #[tokio::test]
18616    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18617        crate::run::pin_test_home();
18618        let mut runner = runner_at(RunStatus::Reviewing);
18619        runner.state.config.agents = vec![conductor()];
18620        runner.state.save().unwrap();
18621        let id = runner.state.id.clone();
18622        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18623        let mut resumed = Runner::resume(&id).unwrap();
18624        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18625        assert!(resumed.claim_driver().await.is_err());
18626        assert!(RunState::load(&id).unwrap().passes.is_empty());
18627        drop(lease);
18628        let mut execution = resumed.claim_driver().await.unwrap();
18629        execution
18630            .driver
18631            .finish(&mut execution.runner.state)
18632            .await
18633            .unwrap();
18634    }
18635
18636    #[tokio::test]
18637    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18638        crate::run::pin_test_home();
18639        let mut runner = runner_at(RunStatus::Reviewing);
18640        let state = &mut runner.state;
18641        state.config.agents = vec![conductor()];
18642        state.driver_claimed = None;
18643        state.driver_pid = Some(std::process::id());
18644        state.driver_started_at = None;
18645        state.begin_pass();
18646        state.save().unwrap();
18647        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18648        std::fs::write(
18649            state.dir().join("driver.json"),
18650            serde_json::to_vec(&serde_json::json!({
18651                "pid": state.driver_pid, "started_at": null,
18652                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18653            }))
18654            .unwrap(),
18655        )
18656        .unwrap();
18657        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18658        drop(lease);
18659        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18660        let mut resumed = Runner::resume(&state.id).unwrap();
18661        assert!(!resumed.state.driver_exited);
18662        let mut execution = resumed
18663            .claim_driver()
18664            .await
18665            .expect("lease proves previous execution ended");
18666        assert_eq!(execution.runner.state.passes.len(), 2);
18667        execution
18668            .driver
18669            .finish(&mut execution.runner.state)
18670            .await
18671            .unwrap();
18672    }
18673
18674    #[tokio::test]
18675    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18676        crate::run::pin_test_home();
18677        let mut runner = runner_at(RunStatus::Reviewing);
18678        let mut json = serde_json::to_value(&runner.state).unwrap();
18679        json.as_object_mut().unwrap().remove("driver_claimed");
18680        runner.state = serde_json::from_value(json).unwrap();
18681        assert_eq!(runner.state.driver_claimed, None);
18682        runner.state.save().unwrap();
18683        let mut execution = runner
18684            .claim_driver()
18685            .await
18686            .expect("the execution lease, not a missing marker, excludes a second driver");
18687        assert_eq!(execution.runner.state.passes.len(), 1);
18688        execution
18689            .driver
18690            .finish(&mut execution.runner.state)
18691            .await
18692            .unwrap();
18693    }
18694
18695    #[tokio::test]
18696    async fn driver_admission_refuses_a_proven_live_driver() {
18697        crate::run::pin_test_home();
18698        let mut runner = runner_at(RunStatus::Reviewing);
18699        let pid = std::process::id();
18700        runner.state.driver_pid = Some(pid);
18701        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18702        assert!(runner.state.driver_started_at.is_some());
18703        runner.state.save().unwrap();
18704        let error = runner
18705            .claim_driver()
18706            .await
18707            .err()
18708            .expect("a live driver must refuse");
18709        assert!(error.to_string().contains("live driver"), "{error:#}");
18710        let saved = RunState::load(&runner.state.id).unwrap();
18711        assert_eq!(saved.driver_pid, Some(pid));
18712        assert_eq!(saved.status, RunStatus::Reviewing);
18713        assert!(saved.passes.is_empty());
18714    }
18715
18716    #[test]
18717    fn parking_resource_halt_persists_the_pass_once() {
18718        crate::run::pin_test_home();
18719        let mut runner = runner_at(RunStatus::Reviewing);
18720        runner.state.begin_pass();
18721        runner.state.resource_block = Some(crate::run::ResourceBlock {
18722            node: "fix".into(),
18723            kind: "shared build cache".into(),
18724            detail: "cancelled".into(),
18725            at: Timestamp::now(),
18726            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18727        });
18728        let cause = ParkCause::Upgrade {
18729            from: "old".into(),
18730            to: Some("new".into()),
18731        };
18732        runner.pause.park_for(cause.clone());
18733        // Both direct fixer exits and graph resource boundaries must complete
18734        // parking, rather than returning a stale Reviewing record.
18735        runner.stop_for_resource().unwrap();
18736        assert!(runner.resource_halt().unwrap());
18737        assert!(runner.park_here().unwrap());
18738        let saved = RunState::load(&runner.state.id).unwrap();
18739        assert!(saved.parked);
18740        assert_eq!(saved.status, RunStatus::Reviewing);
18741        assert!(saved.passes[0].ended_at.is_some());
18742        assert_eq!(
18743            saved.passes[0].end,
18744            Some(crate::run::PassEnd::Parked {
18745                cause: cause.clone()
18746            })
18747        );
18748        assert_eq!(
18749            saved.resource_block.unwrap().cause,
18750            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18751        );
18752        assert_eq!(
18753            saved
18754                .events
18755                .iter()
18756                .filter(|event| event.node == "park")
18757                .count(),
18758            1
18759        );
18760    }
18761
18762    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
18763    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
18764    /// cleared) must not make a *different* run - one only watching its own,
18765    /// unshared interrupt `Pause` - see itself as parked. If a future change
18766    /// ever collapsed these back into one handle, the interrupt scheduler
18767    /// would park every run for the rest of the daemon's life, not just the
18768    /// one it meant to interrupt.
18769    #[test]
18770    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18771        crate::run::pin_test_home();
18772        let mut runner = runner_at(RunStatus::Implementing);
18773        let shutdown = Pause::new();
18774        runner.on_pause(shutdown.clone());
18775        let interrupt = Pause::new();
18776        runner.watch_interrupt(interrupt.clone());
18777
18778        // Nobody has asked for anything yet.
18779        assert!(!runner.park_here().expect("park_here"));
18780        assert!(!runner.state.parked);
18781
18782        // Only the interrupt handle fires; the shutdown handle stays clear.
18783        interrupt.park_because("test");
18784        assert!(!shutdown.parked());
18785        assert!(runner.park_here().expect("park_here"));
18786    }
18787
18788    /// The property every prior attempt at this feature failed to pin down:
18789    /// asking a run to park while one of its nodes has a real, in-flight
18790    /// async operation running (an agent call, in production) must not cut
18791    /// that operation short. `park_here` is only ever consulted *between*
18792    /// `execute`'s node calls - see its own doc - so nothing inside a node
18793    /// can observe a park request until the node itself returns. This proves
18794    /// that structurally, with real `tokio` concurrency and a channel
18795    /// handshake (never a sleep, which would only prove "usually", not
18796    /// "cannot"): the "node" below reports that it has genuinely started,
18797    /// and only then is the park requested; the node still has to be told to
18798    /// finish before `park_here` is ever called, exactly mirroring every
18799    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
18800    /// in `execute`.
18801    #[tokio::test]
18802    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18803        crate::run::pin_test_home();
18804        let mut runner = runner_at(RunStatus::Implementing);
18805        let interrupt = Pause::new();
18806        runner.watch_interrupt(interrupt.clone());
18807
18808        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18809        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18810
18811        // Stands in for one node's in-flight agent call: it proves it has
18812        // genuinely started, then blocks - exactly as a spawned CLI process
18813        // does - until told to finish.
18814        let node = async move {
18815            started_tx.send(()).expect("send started");
18816            finish_rx.await.expect("recv finish");
18817            "node finished"
18818        };
18819
18820        let interrupter = async move {
18821            started_rx.await.expect("recv started");
18822            // The call is now genuinely in flight. Ask it to park.
18823            interrupt.park_because("higher-priority task waiting");
18824            // Nothing the node does can observe this yet - there is no
18825            // check inside it, by construction - so let the executor run
18826            // anything pending and then let the node finish on its own.
18827            tokio::task::yield_now().await;
18828            finish_tx.send(()).expect("send finish");
18829        };
18830
18831        let (node_result, ()) = tokio::join!(node, interrupter);
18832        assert_eq!(
18833            node_result, "node finished",
18834            "the in-flight call ran to completion"
18835        );
18836
18837        // Only now, at the boundary the real `execute` would check right
18838        // after this node, does the park take effect.
18839        assert!(runner.park_here().expect("park_here"));
18840        assert!(runner.state.parked);
18841    }
18842
18843    /// A run parked mid-competition carries every field it had accumulated
18844    /// through the exact same disk round-trip an ordinary resume uses -
18845    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
18846    /// Nothing about parking for an interrupt is a special case of that path;
18847    /// this is what proves it rather than assuming it.
18848    #[test]
18849    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18850        crate::run::pin_test_home();
18851        let mut runner = runner_at(RunStatus::Judging);
18852        // `Runner::resume` re-resolves roles from the saved config, which
18853        // refuses an empty roster - give it the same minimal one `conductor`
18854        // itself uses.
18855        runner.state.config.agents = vec![conductor()];
18856        runner.state.candidates = vec![Candidate {
18857            index: 0,
18858            label: 'A',
18859            agent: "alpha".to_owned(),
18860            branch: "magi/x/A".to_owned(),
18861            worktree: PathBuf::from("/nonexistent/worktree"),
18862            summary: "did the thing".to_owned(),
18863            stat: "1 file changed".to_owned(),
18864            files: 1,
18865            commits: 1,
18866            empty: false,
18867            failed: None,
18868            verified_noop: None,
18869            duration_ms: 1234,
18870            folded: false,
18871        }];
18872        let run_id = runner.state.id.clone();
18873
18874        let interrupt = Pause::new();
18875        runner.watch_interrupt(interrupt.clone());
18876        interrupt.park_because("task c3d4 asked to run first");
18877        assert!(runner.park_here().expect("park_here"));
18878
18879        let resumed = Runner::resume(&run_id).expect("resume");
18880        assert_eq!(resumed.state.candidates.len(), 1);
18881        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18882        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18883        assert_eq!(resumed.state.status, runner.state.status);
18884        assert!(
18885            resumed.state.parked,
18886            "still parked until `execute` actually walks the graph again"
18887        );
18888        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18889    }
18890
18891    /// A fresh open question on `run`, stored and handed back for assertions.
18892    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18893        let mut q = ask::Question::new(
18894            run.to_owned(),
18895            "implement".to_owned(),
18896            "impl-A".to_owned(),
18897            "Which storage backend should the cache use?".to_owned(),
18898            String::new(),
18899            vec!["SQLite".to_owned(), "Redis".to_owned()],
18900        );
18901        store.put(&mut q).unwrap();
18902        q
18903    }
18904
18905    #[test]
18906    fn a_failed_runs_open_question_is_abandoned() {
18907        ask_test_home();
18908        let store = ask::Questions::open();
18909        let mut runner = runner_at(RunStatus::Failed);
18910        let run = runner.state.id.clone();
18911        let q = ask_open_question(&store, &run);
18912
18913        runner.settle_questions();
18914
18915        let back = store.get(&q.id).unwrap();
18916        assert!(
18917            !back.status.open(),
18918            "the seat that asked died with the run; nobody is left to read an answer"
18919        );
18920        assert!(
18921            back.detail.contains(&run) && back.detail.contains("failed"),
18922            "the reason names what the run became, not just that it is gone: {}",
18923            back.detail
18924        );
18925    }
18926
18927    #[test]
18928    fn a_merged_runs_open_question_is_abandoned_too() {
18929        ask_test_home();
18930        let store = ask::Questions::open();
18931        // A run that finishes cleanly still leaves nobody to read an answer -
18932        // this is not only a failure-path cleanup.
18933        for status in [RunStatus::Merged, RunStatus::Ready] {
18934            let mut runner = runner_at(status);
18935            let run = runner.state.id.clone();
18936            let q = ask_open_question(&store, &run);
18937
18938            runner.settle_questions();
18939
18940            let back = store.get(&q.id).unwrap();
18941            assert!(
18942                !back.status.open(),
18943                "{status:?} run's question must not outlive the run"
18944            );
18945        }
18946    }
18947
18948    #[test]
18949    fn a_still_resumable_runs_open_question_is_left_alone() {
18950        ask_test_home();
18951        let store = ask::Questions::open();
18952        // `Blocked` and `Stalled` can still be resumed — the candidates, the
18953        // review round and the seat sessions are all still on disk — so a
18954        // question asked mid-round may yet get a real answer from a real
18955        // resume. Sweeping it here would be exactly the failure mode this
18956        // whole feature exists to avoid on the other side.
18957        for status in [RunStatus::Blocked, RunStatus::Stalled] {
18958            let mut runner = runner_at(status);
18959            let run = runner.state.id.clone();
18960            let q = ask_open_question(&store, &run);
18961
18962            runner.settle_questions();
18963
18964            let back = store.get(&q.id).unwrap();
18965            assert!(
18966                back.status.open(),
18967                "{status:?} is still alive; the question must still be waiting"
18968            );
18969        }
18970    }
18971
18972    #[test]
18973    fn settle_questions_never_touches_an_already_answered_question() {
18974        ask_test_home();
18975        let store = ask::Questions::open();
18976        let mut runner = runner_at(RunStatus::Failed);
18977        let run = runner.state.id.clone();
18978        let mut q = ask_open_question(&store, &run);
18979        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
18980            .unwrap();
18981        store.put(&mut q).unwrap();
18982
18983        // Called twice, the way a crash-recovered daemon reclaim and the
18984        // graph's own cleanup both can for the same run — `abandon_for_run`
18985        // only ever touches what is still open, so this must be inert both
18986        // times, not merely the second.
18987        runner.settle_questions();
18988        runner.settle_questions();
18989
18990        let back = store.get(&q.id).unwrap();
18991        assert_eq!(
18992            back.status,
18993            ask::QuestionStatus::Answered,
18994            "a real answer is a decision on record, never overwritten by a sweep"
18995        );
18996    }
18997
18998    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
18999    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19000    /// without merging, whose winner is still the operator's answer to read.
19001    /// Nothing previously called `fold_run` itself with a real `tally`, so
19002    /// this is the first test to pin down the one distinction the whole
19003    /// automatic-fold feature depends on: the winner's worktree and branch
19004    /// must survive, everything else sharing the run's worktree bay - a
19005    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19006    /// second sweep treats every non-winner directory under the bay alike -
19007    /// must not.
19008    #[tokio::test]
19009    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19010        crate::run::pin_test_home();
19011        let tmp = tempfile::tempdir().expect("tempdir");
19012        let repo = tmp.path().join("repo");
19013        std::fs::create_dir_all(&repo).unwrap();
19014        init_repo(&repo);
19015
19016        let mut config = Config::default();
19017        config.graph.worktree_root = Some(tmp.path().join("wt"));
19018
19019        let mut state = RunState::new(
19020            repo.clone(),
19021            "main".to_owned(),
19022            "deadbeef".to_owned(),
19023            "task".to_owned(),
19024            config,
19025        );
19026        let root = state.worktree_root();
19027        let wt_a = root.join("cand-A");
19028        let wt_b = root.join("cand-B");
19029        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19030            .await
19031            .expect("worktree A");
19032        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19033            .await
19034            .expect("worktree B");
19035
19036        state.candidates = vec![
19037            Candidate {
19038                index: 0,
19039                label: 'A',
19040                agent: "alpha".to_owned(),
19041                branch: "magi/x/A".to_owned(),
19042                worktree: wt_a.clone(),
19043                summary: String::new(),
19044                stat: String::new(),
19045                files: 0,
19046                commits: 0,
19047                empty: false,
19048                failed: None,
19049                verified_noop: None,
19050                duration_ms: 0,
19051                folded: false,
19052            },
19053            Candidate {
19054                index: 1,
19055                label: 'B',
19056                agent: "beta".to_owned(),
19057                branch: "magi/x/B".to_owned(),
19058                worktree: wt_b.clone(),
19059                summary: String::new(),
19060                stat: String::new(),
19061                files: 0,
19062                commits: 0,
19063                empty: false,
19064                failed: None,
19065                verified_noop: None,
19066                duration_ms: 0,
19067                folded: false,
19068            },
19069        ];
19070        state.tally = Some(Tally {
19071            first_choice: BTreeMap::from([('A', 1)]),
19072            borda: BTreeMap::new(),
19073            winner: 'A',
19074            rankings: 1,
19075            unanimous_initial: true,
19076            deliberated: false,
19077            changed_votes: 0,
19078            unanimous_final: true,
19079            tie_break: None,
19080            judges: 1,
19081            present: 1,
19082            quorum: 1,
19083            met_quorum: true,
19084            uncontested: None,
19085        });
19086        state.status = RunStatus::Ready;
19087
19088        fold_run(&mut state, false, &crate::run::home())
19089            .await
19090            .expect("fold_run");
19091
19092        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19093        assert!(
19094            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19095            "the unmerged winner's branch survives"
19096        );
19097        assert!(
19098            !state.candidates[0].folded,
19099            "the winner is not marked folded"
19100        );
19101
19102        assert!(!wt_b.exists(), "the loser's worktree is removed");
19103        assert!(
19104            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19105            "the loser's branch is removed"
19106        );
19107        assert!(state.candidates[1].folded, "the loser is marked folded");
19108    }
19109
19110    #[tokio::test]
19111    #[cfg(unix)]
19112    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19113        use std::os::fd::AsRawFd as _;
19114        use std::time::Duration;
19115
19116        let tmp = tempfile::tempdir().unwrap();
19117        let repo = tmp.path().join("repo");
19118        std::fs::create_dir_all(&repo).unwrap();
19119        init_repo(&repo);
19120        let common = repo.join(".git");
19121        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19122        let home = tmp.path().join("home");
19123        let mut config = Config::default();
19124        config.coordination = crate::config::test_coordination();
19125        config.graph.worktree_root = Some(tmp.path().join("wt"));
19126        config.graph.timeout_resource_wait = 1;
19127        let mut state = RunState::new(
19128            repo.clone(),
19129            "main".into(),
19130            "deadbeef".into(),
19131            "task".into(),
19132            config,
19133        );
19134        let root = state.worktree_root();
19135        let candidate = root.join("cand-A");
19136        let reviewer = root.join("review-1");
19137        let branch = state.branch_for('A');
19138        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19139            .await
19140            .unwrap();
19141        git::worktree_add_detached(&repo, &reviewer, "main")
19142            .await
19143            .unwrap();
19144        git::acquire_worktree_config(&repo).await.unwrap();
19145        state.enabled_worktree_config = true;
19146        state.status = RunStatus::Ready;
19147        state.candidates.push(Candidate {
19148            index: 0,
19149            label: 'A',
19150            agent: "implement".into(),
19151            branch: branch.clone(),
19152            worktree: candidate.clone(),
19153            summary: String::new(),
19154            stat: String::new(),
19155            files: 0,
19156            commits: 0,
19157            empty: false,
19158            failed: None,
19159            verified_noop: None,
19160            duration_ms: 0,
19161            folded: false,
19162        });
19163        state.save_under(&home).unwrap();
19164        let config_before = std::fs::read(common.join("config")).unwrap();
19165        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19166            .await
19167            .unwrap();
19168        // A separate OS handle bypasses the launcher's local gate, exactly as
19169        // an external updater does, and must still prevent all folding work.
19170        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19171        assert_eq!(
19172            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19173            0
19174        );
19175        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19176        assert!(
19177            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19178                .await
19179                .is_err()
19180        );
19181        assert!(candidate.exists());
19182        assert!(reviewer.exists());
19183        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19184        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19185        assert_eq!(
19186            git::git(&repo, &["worktree", "list", "--porcelain"])
19187                .await
19188                .unwrap(),
19189            worktrees_before
19190        );
19191        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19192            .await
19193            .unwrap()
19194            .unwrap_err();
19195        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19196        drop(fold);
19197        assert!(candidate.exists() && reviewer.exists());
19198        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19199        assert_eq!(
19200            state.resource_block.as_ref().unwrap().kind,
19201            "repository coordination"
19202        );
19203        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19204        drop(external);
19205        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19206        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19207            .await
19208            .unwrap()
19209            .unwrap();
19210        drop(fold);
19211        assert!(removed.contains(&branch));
19212        assert!(!root.exists());
19213        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19214        assert!(state.candidates[0].folded);
19215        assert!(!state.enabled_worktree_config);
19216        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19217        let enabled = git::git_raw(
19218            &repo,
19219            &["config", "--local", "--get", "extensions.worktreeConfig"],
19220        )
19221        .await
19222        .unwrap();
19223        assert!(!enabled.ok(), "the last config reference was released");
19224    }
19225
19226    /// A branch handed to a later run is that run's (and its pull request's):
19227    /// folding the run that released it must not delete it.
19228    #[tokio::test]
19229    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19230        let tmp = tempfile::tempdir().expect("tempdir");
19231        let repo = tmp.path().join("repo");
19232        std::fs::create_dir_all(&repo).unwrap();
19233        init_repo(&repo);
19234        let home = tmp.path().join("home");
19235
19236        let mut config = Config::default();
19237        config.graph.worktree_root = Some(tmp.path().join("wt"));
19238        let mut state = RunState::new(
19239            repo.clone(),
19240            "main".to_owned(),
19241            "deadbeef".to_owned(),
19242            "task".to_owned(),
19243            config,
19244        );
19245        // The worktree is already gone (released); the branch survives.
19246        git::git(&repo, &["branch", "magi/x/A", "main"])
19247            .await
19248            .expect("branch");
19249        state.candidates = vec![Candidate {
19250            index: 0,
19251            label: 'A',
19252            agent: "alpha".to_owned(),
19253            branch: "magi/x/A".to_owned(),
19254            worktree: state.worktree_root().join("cand-A"),
19255            summary: String::new(),
19256            stat: String::new(),
19257            files: 0,
19258            commits: 0,
19259            empty: false,
19260            failed: None,
19261            verified_noop: None,
19262            duration_ms: 0,
19263            folded: true,
19264        }];
19265        state.released_to = Some("20260901-000000-new1".to_owned());
19266        state.released_branches = vec!["magi/x/A".to_owned()];
19267
19268        fold_run(&mut state, true, &home).await.expect("fold_run");
19269        state.status = RunStatus::Merged;
19270        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19271        state.merge = Some(crate::run::MergeOutcome {
19272            mode: MergeMode::Pr,
19273            ok: true,
19274            detail: "confirmed merged".into(),
19275            empty: false,
19276        });
19277        fold_run(&mut state, true, &home)
19278            .await
19279            .expect("merged fold");
19280
19281        assert!(
19282            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19283            "the handed-over branch survives a fold"
19284        );
19285    }
19286
19287    /// A winner with nothing ahead of the base is caught before `gh` is ever
19288    /// asked for a pull request, and the message carries what the task's
19289    /// references resolved to.
19290    #[tokio::test]
19291    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19292        let tmp = tempfile::tempdir().expect("tempdir");
19293        let repo = tmp.path().join("repo");
19294        std::fs::create_dir_all(&repo).unwrap();
19295        init_repo(&repo);
19296        let run = |args: &[&str]| {
19297            let out = std::process::Command::new("git")
19298                .quiet()
19299                .args(args)
19300                .current_dir(&repo)
19301                .output()
19302                .expect("spawn git");
19303            assert!(out.status.success(), "git {args:?}");
19304        };
19305        run(&["branch", "magi/x/A"]);
19306        run(&["checkout", "-q", "-b", "magi/x/B"]);
19307        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19308        run(&["add", "-A"]);
19309        run(&["commit", "-q", "-m", "work"]);
19310        run(&["checkout", "-q", "main"]);
19311
19312        // A pull request is compared against the remote's base, so the
19313        // fixture needs one. Before it exists nothing can be read, and the
19314        // answer must be "not empty".
19315        let probe = RunState::new(
19316            repo.clone(),
19317            "main".to_owned(),
19318            "deadbeef".to_owned(),
19319            "task".to_owned(),
19320            Config::default(),
19321        );
19322        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19323        let bare = tmp.path().join("origin.git");
19324        let out = std::process::Command::new("git")
19325            .quiet()
19326            .args(["init", "-q", "--bare"])
19327            .arg(&bare)
19328            .output()
19329            .expect("spawn git");
19330        assert!(out.status.success(), "git init --bare");
19331        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19332        run(&["push", "-q", "origin", "main"]);
19333
19334        let mut state = RunState::new(
19335            repo.clone(),
19336            "main".to_owned(),
19337            "deadbeef".to_owned(),
19338            "task".to_owned(),
19339            Config::default(),
19340        );
19341        state.seeds = vec![refs::Seed {
19342            token: "magi/27b2/A".to_owned(),
19343            kind: refs::SeedKind::Unresolved,
19344            sha: String::new(),
19345            branch: true,
19346            detail: "no branch or commit named magi/27b2/A".to_owned(),
19347        }];
19348
19349        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19350        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19351        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19352        let detail = empty_candidate_detail(&state, "main");
19353        assert!(detail.starts_with("empty candidate"), "{detail}");
19354        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19355        assert!(detail.contains("--start-from"), "{detail}");
19356    }
19357
19358    /// `status == Ready` used to be read as "this is the harmless
19359    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19360    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19361    /// run's PR was closed without merging — and reentering `merge` with
19362    /// `mode` still `Pr` does not know the difference, so it pushed and
19363    /// opened a second pull request. `mode == Local` reproduces the same
19364    /// blind spot without a network call: reentry must not attempt another
19365    /// git merge once this node has already recorded an outcome.
19366    #[tokio::test]
19367    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19368        let tmp = tempfile::tempdir().expect("tempdir");
19369        let repo = tmp.path().join("repo");
19370        std::fs::create_dir_all(&repo).unwrap();
19371        init_repo(&repo);
19372
19373        let mut config = Config::default();
19374        config.merge.mode = MergeMode::Local;
19375
19376        let mut state = RunState::new(
19377            repo.clone(),
19378            "main".to_owned(),
19379            "deadbeef".to_owned(),
19380            "task".to_owned(),
19381            config,
19382        );
19383        state.candidates = vec![Candidate {
19384            index: 0,
19385            label: 'A',
19386            agent: "alpha".to_owned(),
19387            branch: "does-not-exist".to_owned(),
19388            worktree: repo.clone(),
19389            summary: String::new(),
19390            stat: String::new(),
19391            files: 0,
19392            commits: 0,
19393            empty: false,
19394            failed: None,
19395            verified_noop: None,
19396            duration_ms: 0,
19397            folded: false,
19398        }];
19399        state.tally = Some(Tally {
19400            first_choice: BTreeMap::from([('A', 1)]),
19401            borda: BTreeMap::new(),
19402            winner: 'A',
19403            rankings: 1,
19404            unanimous_initial: true,
19405            deliberated: false,
19406            changed_votes: 0,
19407            unanimous_final: true,
19408            tie_break: None,
19409            judges: 0,
19410            present: 0,
19411            quorum: 0,
19412            met_quorum: true,
19413            uncontested: Some("only candidate A produced a change".to_owned()),
19414        });
19415        state.reviews = vec![ReviewRound {
19416            round: 1,
19417            head: "deadbeef".to_owned(),
19418            verified_head: None,
19419            verified_at: None,
19420            reviews: Vec::new(),
19421            e2e: Vec::new(),
19422            fix: None,
19423            blocking: 0,
19424            answered: 0,
19425            expected: 0,
19426            clean: true,
19427            verify_retried: false,
19428            e2e_deferred: false,
19429            e2e_defer_reason: None,
19430            progressed: false,
19431            vote_split: false,
19432            reconsideration: Vec::new(),
19433            verdict: None,
19434        }];
19435        state.gate = vec![CommandOutcome {
19436            command: "test".to_owned(),
19437            code: Some(0),
19438            output_tail: String::new(),
19439            duration_ms: 0,
19440            resource_blocked: false,
19441        }];
19442        state.gate_ran = true;
19443        // Reached its conclusion already — e.g. `land` closing the PR without
19444        // merging it, which (like the honest `MergeMode::None` path) leaves
19445        // `status` at `Ready`. The recorded outcome is what actually marks
19446        // this node done.
19447        state.status = RunStatus::Ready;
19448        state.merge = Some(MergeOutcome {
19449            mode: MergeMode::Local,
19450            ok: false,
19451            detail: "already concluded".to_owned(),
19452            empty: false,
19453        });
19454
19455        let mut runner = Runner {
19456            state,
19457            roles: ResolvedRoles {
19458                implementers: Vec::new(),
19459                judges: Vec::new(),
19460                reviewers: Vec::new(),
19461                fixer: None,
19462                conductor: conductor(),
19463                implementer_roster: Vec::new(),
19464                judge_roster: Vec::new(),
19465                reviewer_roster: Vec::new(),
19466            },
19467            sem: Arc::new(Semaphore::new(1)),
19468            pause: Pause::new(),
19469            interrupt: Pause::new(),
19470        };
19471
19472        runner.merge().await.expect("merge");
19473
19474        assert_eq!(
19475            runner.state.status,
19476            RunStatus::Ready,
19477            "a concluded run's status must not change on reentry"
19478        );
19479        assert_eq!(
19480            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19481            Some("already concluded"),
19482            "merge must not run again once the node already recorded an outcome"
19483        );
19484    }
19485
19486    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19487    /// when its last attempt was resource-blocked (the shared build cache
19488    /// could not be acquired or confirmed fresh in time - see
19489    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19490    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19491    /// a run merge a tree the gate never actually checked - exactly the case
19492    /// a contended cache produces on every retry until it clears. `merge`
19493    /// must refuse until `gate` has actually recorded an attempt.
19494    #[tokio::test]
19495    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19496        let tmp = tempfile::tempdir().expect("tempdir");
19497        let repo = tmp.path().join("repo");
19498        std::fs::create_dir_all(&repo).unwrap();
19499        init_repo(&repo);
19500
19501        let mut config = Config::default();
19502        config.merge.mode = MergeMode::Local;
19503
19504        let mut state = RunState::new(
19505            repo.clone(),
19506            "main".to_owned(),
19507            "deadbeef".to_owned(),
19508            "task".to_owned(),
19509            config,
19510        );
19511        state.candidates = vec![Candidate {
19512            index: 0,
19513            label: 'A',
19514            agent: "alpha".to_owned(),
19515            branch: "does-not-exist".to_owned(),
19516            worktree: repo.clone(),
19517            summary: String::new(),
19518            stat: String::new(),
19519            files: 0,
19520            commits: 0,
19521            empty: false,
19522            failed: None,
19523            verified_noop: None,
19524            duration_ms: 0,
19525            folded: false,
19526        }];
19527        state.tally = Some(Tally {
19528            first_choice: BTreeMap::from([('A', 1)]),
19529            borda: BTreeMap::new(),
19530            winner: 'A',
19531            rankings: 1,
19532            unanimous_initial: true,
19533            deliberated: false,
19534            changed_votes: 0,
19535            unanimous_final: true,
19536            tie_break: None,
19537            judges: 0,
19538            present: 0,
19539            quorum: 0,
19540            met_quorum: true,
19541            uncontested: Some("only candidate A produced a change".to_owned()),
19542        });
19543        state.reviews = vec![ReviewRound {
19544            round: 1,
19545            head: "deadbeef".to_owned(),
19546            verified_head: None,
19547            verified_at: None,
19548            reviews: Vec::new(),
19549            e2e: Vec::new(),
19550            fix: None,
19551            blocking: 0,
19552            answered: 0,
19553            expected: 0,
19554            clean: true,
19555            verify_retried: false,
19556            e2e_deferred: false,
19557            e2e_defer_reason: None,
19558            progressed: false,
19559            vote_split: false,
19560            reconsideration: Vec::new(),
19561            verdict: None,
19562        }];
19563        // The point: `gate` has not recorded anything yet.
19564        state.gate = Vec::new();
19565        state.gate_ran = false;
19566        state.status = RunStatus::Gating;
19567
19568        let mut runner = Runner {
19569            state,
19570            roles: ResolvedRoles {
19571                implementers: Vec::new(),
19572                judges: Vec::new(),
19573                reviewers: Vec::new(),
19574                fixer: None,
19575                conductor: conductor(),
19576                implementer_roster: Vec::new(),
19577                judge_roster: Vec::new(),
19578                reviewer_roster: Vec::new(),
19579            },
19580            sem: Arc::new(Semaphore::new(1)),
19581            pause: Pause::new(),
19582            interrupt: Pause::new(),
19583        };
19584
19585        runner.merge().await.expect("merge");
19586
19587        assert!(
19588            runner.state.merge.is_none(),
19589            "an empty gate must never be read as a passing one: {:?}",
19590            runner.state.merge
19591        );
19592    }
19593
19594    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19595    /// commands configured and `merge.mode` is `none` (a review-only run).
19596    /// `gate` must still record a real attempt — zero commands, vacuously
19597    /// passed — rather than leaving `state.gate` empty in a way `merge`
19598    /// cannot tell apart from "never ran"; otherwise the run reaches
19599    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19600    #[tokio::test]
19601    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19602        let tmp = tempfile::tempdir().expect("tempdir");
19603        let repo = tmp.path().join("repo");
19604        std::fs::create_dir_all(&repo).unwrap();
19605        init_repo(&repo);
19606
19607        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19608        let config = Config::default();
19609
19610        let mut state = RunState::new(
19611            repo.clone(),
19612            "main".to_owned(),
19613            "deadbeef".to_owned(),
19614            "task".to_owned(),
19615            config,
19616        );
19617        state.candidates = vec![Candidate {
19618            index: 0,
19619            label: 'A',
19620            agent: "alpha".to_owned(),
19621            branch: "does-not-exist".to_owned(),
19622            worktree: repo.clone(),
19623            summary: String::new(),
19624            stat: String::new(),
19625            files: 0,
19626            commits: 0,
19627            empty: false,
19628            failed: None,
19629            verified_noop: None,
19630            duration_ms: 0,
19631            folded: false,
19632        }];
19633        state.tally = Some(Tally {
19634            first_choice: BTreeMap::from([('A', 1)]),
19635            borda: BTreeMap::new(),
19636            winner: 'A',
19637            rankings: 1,
19638            unanimous_initial: true,
19639            deliberated: false,
19640            changed_votes: 0,
19641            unanimous_final: true,
19642            tie_break: None,
19643            judges: 0,
19644            present: 0,
19645            quorum: 0,
19646            met_quorum: true,
19647            uncontested: Some("only candidate A produced a change".to_owned()),
19648        });
19649        state.reviews = vec![ReviewRound {
19650            round: 1,
19651            head: "deadbeef".to_owned(),
19652            verified_head: None,
19653            verified_at: None,
19654            reviews: Vec::new(),
19655            e2e: Vec::new(),
19656            fix: None,
19657            blocking: 0,
19658            answered: 0,
19659            expected: 0,
19660            clean: true,
19661            verify_retried: false,
19662            e2e_deferred: false,
19663            e2e_defer_reason: None,
19664            progressed: false,
19665            vote_split: false,
19666            reconsideration: Vec::new(),
19667            verdict: None,
19668        }];
19669
19670        let mut runner = Runner {
19671            state,
19672            roles: ResolvedRoles {
19673                implementers: Vec::new(),
19674                judges: Vec::new(),
19675                reviewers: Vec::new(),
19676                fixer: None,
19677                conductor: conductor(),
19678                implementer_roster: Vec::new(),
19679                judge_roster: Vec::new(),
19680                reviewer_roster: Vec::new(),
19681            },
19682            sem: Arc::new(Semaphore::new(1)),
19683            pause: Pause::new(),
19684            interrupt: Pause::new(),
19685        };
19686
19687        runner.gate().await.expect("gate");
19688        assert!(
19689            runner.state.gate_ran,
19690            "zero configured commands is still a real attempt, not an unrun gate"
19691        );
19692        assert!(runner.state.gate.is_empty());
19693        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19694        assert_ne!(
19695            runner.state.status,
19696            RunStatus::Blocked,
19697            "a gate with nothing to check must not read as failed"
19698        );
19699
19700        runner.merge().await.expect("merge");
19701        assert_eq!(
19702            runner.state.status,
19703            RunStatus::Ready,
19704            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19705        );
19706    }
19707
19708    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19709    /// `verify.gate` (so the e2e leg and the final gate never build against
19710    /// different directories). With zero `verify.gate` commands but a
19711    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19712    /// that lease before discovering it had nothing to run - so a repo with
19713    /// no gate commands could come back `resource_blocked` (and therefore
19714    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19715    /// cache, exactly the contention this run's own zero commands could
19716    /// never have touched. `gate` must recognise there is nothing to check
19717    /// before it ever asks for the lease.
19718    #[tokio::test]
19719    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19720        crate::run::pin_test_home();
19721        let home = crate::run::home();
19722
19723        let tmp = tempfile::tempdir().expect("tempdir");
19724        let repo = tmp.path().join("repo");
19725        std::fs::create_dir_all(&repo).unwrap();
19726        init_repo(&repo);
19727        // Unique to this test, so holding its lease cannot collide with
19728        // another test sharing the same process-wide `home`.
19729        let cache_dir = tmp.path().join("target");
19730
19731        let mut config = Config::default();
19732        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19733        // `verify.gate` stays empty (the default). Any accidental cache
19734        // acquisition must report contention immediately, regardless of how
19735        // long unrelated Git subprocesses take under parallel test load.
19736        config.graph.timeout_verify = Some(2);
19737        config.graph.timeout_resource_wait = 0;
19738
19739        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19740        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19741            .expect("no io error acquiring directly")
19742        {
19743            crate::cache::AcquireOutcome::Acquired(g) => g,
19744            crate::cache::AcquireOutcome::Busy(b) => {
19745                panic!("expected the direct acquire to win the lease first: {b:?}")
19746            }
19747        };
19748
19749        let mut state = RunState::new(
19750            repo.clone(),
19751            "main".to_owned(),
19752            "deadbeef".to_owned(),
19753            "task".to_owned(),
19754            config,
19755        );
19756        state.candidates = vec![Candidate {
19757            index: 0,
19758            label: 'A',
19759            agent: "alpha".to_owned(),
19760            branch: "does-not-exist".to_owned(),
19761            worktree: repo.clone(),
19762            summary: String::new(),
19763            stat: String::new(),
19764            files: 0,
19765            commits: 0,
19766            empty: false,
19767            failed: None,
19768            verified_noop: None,
19769            duration_ms: 0,
19770            folded: false,
19771        }];
19772        state.tally = Some(Tally {
19773            first_choice: BTreeMap::from([('A', 1)]),
19774            borda: BTreeMap::new(),
19775            winner: 'A',
19776            rankings: 1,
19777            unanimous_initial: true,
19778            deliberated: false,
19779            changed_votes: 0,
19780            unanimous_final: true,
19781            tie_break: None,
19782            judges: 0,
19783            present: 0,
19784            quorum: 0,
19785            met_quorum: true,
19786            uncontested: Some("only candidate A produced a change".to_owned()),
19787        });
19788        state.reviews = vec![ReviewRound {
19789            round: 1,
19790            head: "deadbeef".to_owned(),
19791            verified_head: None,
19792            verified_at: None,
19793            reviews: Vec::new(),
19794            e2e: Vec::new(),
19795            fix: None,
19796            blocking: 0,
19797            answered: 0,
19798            expected: 0,
19799            clean: true,
19800            verify_retried: false,
19801            e2e_deferred: false,
19802            e2e_defer_reason: None,
19803            progressed: false,
19804            vote_split: false,
19805            reconsideration: Vec::new(),
19806            verdict: None,
19807        }];
19808
19809        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19810        state.reviews[0].e2e = vec![CommandOutcome {
19811            command: state.config.verify.e2e[0].clone(),
19812            code: Some(0),
19813            output_tail: String::new(),
19814            duration_ms: 0,
19815            resource_blocked: false,
19816        }];
19817        let mut runner = Runner {
19818            state,
19819            roles: ResolvedRoles {
19820                implementers: Vec::new(),
19821                judges: Vec::new(),
19822                reviewers: Vec::new(),
19823                fixer: None,
19824                conductor: conductor(),
19825                implementer_roster: Vec::new(),
19826                judge_roster: Vec::new(),
19827                reviewer_roster: Vec::new(),
19828            },
19829            sem: Arc::new(Semaphore::new(1)),
19830            pause: Pause::new(),
19831            interrupt: Pause::new(),
19832        };
19833
19834        runner.gate().await.expect("gate");
19835        assert!(runner.state.resource_block.is_none());
19836        assert!(
19837            !runner
19838                .state
19839                .events
19840                .iter()
19841                .any(|event| { event.message.contains("waiting for the shared build cache") }),
19842            "a gate with nothing to run must never request the held cache lease"
19843        );
19844        assert!(
19845            runner.state.gate_ran,
19846            "zero commands is still a real, immediate attempt"
19847        );
19848        assert!(runner.state.gate.is_empty());
19849        assert_ne!(
19850            runner.state.status,
19851            RunStatus::Blocked,
19852            "must not read as resource-blocked on a lease it never asked for"
19853        );
19854    }
19855
19856    /// A separate process carries a live worker's environment throughout all
19857    /// verification entry points; the parallel harness never changes its env.
19858    #[cfg(unix)]
19859    #[tokio::test]
19860    async fn verification_is_standalone_inside_a_live_worker() {
19861        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19862        if std::env::var_os(CHILD).is_none() {
19863            let home = tempfile::tempdir().unwrap();
19864            let generation = crate::worker::pin_current(home.path()).unwrap();
19865            let output = std::process::Command::new(std::env::current_exe().unwrap())
19866                .args([
19867                    "--exact",
19868                    "graph::tests::verification_is_standalone_inside_a_live_worker",
19869                    "--nocapture",
19870                ])
19871                .env(CHILD, "1")
19872                .env("MAGI_HOME", home.path())
19873                .env("MAGI_WORKER_HOME", home.path())
19874                .env("MAGI_WORKER_ID", "abcdef")
19875                .env("MAGI_WORKER_GENERATION", &generation.id)
19876                .env("MAGI_WORKER_EXE", &generation.executable)
19877                .env("MAGI_RUN", "verification-parent")
19878                .env("MAGI_NODE", "gate")
19879                .env("MAGI_SEAT", "worker-fixture")
19880                .env("MAGI_PROCESS_OWNER", "fixture-owner")
19881                .env("MAGI_NO_AUTOUPDATE", "0")
19882                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19883                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19884                .env(
19885                    "VERIFY_EXPECT_TOOLCHAIN",
19886                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19887                )
19888                .env(
19889                    "VERIFY_EXPECT_CACHE",
19890                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19891                )
19892                .output()
19893                .unwrap();
19894            assert!(
19895                output.status.success()
19896                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19897                "{}\n{}",
19898                String::from_utf8_lossy(&output.stdout),
19899                String::from_utf8_lossy(&output.stderr)
19900            );
19901            return;
19902        }
19903        let home = crate::run::home();
19904        let directory = home.join("workers/jobs/abcdef");
19905        std::fs::create_dir_all(&directory).unwrap();
19906        let job = crate::worker::Job {
19907            id: "abcdef".into(),
19908            sequence: 1,
19909            kind: "task".into(),
19910            key: "task:verification".into(),
19911            payload: serde_json::json!({}),
19912            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19913            version: Some(env!("CARGO_PKG_VERSION").into()),
19914            status: crate::worker::Status::Running,
19915            result: None,
19916            pid: Some(std::process::id()),
19917            created_at: 0,
19918            finished_at: None,
19919        };
19920        std::fs::write(
19921            directory.join("job.json"),
19922            serde_json::to_vec(&job).unwrap(),
19923        )
19924        .unwrap();
19925        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19926        let assert_worker = || {
19927            assert_eq!(
19928                crate::worker::current_job(&home).unwrap().unwrap().id,
19929                job.id
19930            );
19931            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19932            assert_eq!(
19933                crate::worker::helper_executable(&home).unwrap(),
19934                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19935            );
19936            assert!(
19937                crate::updater::Checker::new(&crate::config::Update {
19938                    mode: crate::config::UpdateMode::Notify,
19939                    interval: None,
19940                    auto_restart: false,
19941                })
19942                .is_none(),
19943                "the driver remains pinned, even while tests are standalone"
19944            );
19945        };
19946        assert_worker();
19947        let repo = home.join("repo");
19948        std::fs::create_dir_all(&repo).unwrap();
19949        init_repo(&repo);
19950        // Keep storage, seat/control context, containment and tool lookup.
19951        // No inherited values (or credentials) are printed on success.
19952        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
19953            test -z "${MAGI_WORKER_GENERATION+x}" &&
19954            test -z "${MAGI_WORKER_HOME+x}" &&
19955            test -z "${MAGI_WORKER_EXE+x}" &&
19956            test "$MAGI_HOME/repo" -ef . &&
19957            test "$MAGI_RUN" = verification-parent &&
19958            test "$MAGI_NODE" = gate &&
19959            test "$MAGI_SEAT" = worker-fixture &&
19960            test -n "$MAGI_PROCESS_OWNER" &&
19961            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
19962            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
19963            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
19964            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
19965        // Negative control: an ordinary child inherits the contamination.
19966        assert!(
19967            !std::process::Command::new("sh")
19968                .args(["-c", probe])
19969                .current_dir(&repo)
19970                .status()
19971                .unwrap()
19972                .success()
19973        );
19974        let unrelated_home = tempfile::tempdir().unwrap();
19975        assert!(
19976            crate::worker::current_job(unrelated_home.path()).is_err(),
19977            "an inherited worker id cannot be resolved in a unit fixture's home"
19978        );
19979        let mut runner = runner_at(RunStatus::Gating);
19980        runner.state.repo = repo.clone();
19981        runner.state.config.disk.min_free_bytes = 0;
19982        runner.state.config.coordination = crate::config::test_coordination();
19983        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
19984        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
19985        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
19986        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
19987        let winner = Candidate {
19988            index: 0,
19989            label: 'A',
19990            agent: "fixture".into(),
19991            branch: "main".into(),
19992            worktree: repo.clone(),
19993            summary: String::new(),
19994            stat: String::new(),
19995            files: 0,
19996            commits: 0,
19997            empty: false,
19998            failed: None,
19999            verified_noop: None,
20000            duration_ms: 0,
20001            folded: false,
20002        };
20003        let (e2e, retried, pids) = run_e2e_with_retry(
20004            &mut runner.state,
20005            &["sh".into(), "-c".into()],
20006            &[format!("{probe} && printf initial")],
20007            &repo,
20008            Duration::from_secs(10),
20009            "initial",
20010        )
20011        .await;
20012        assert!(!retried && pids.is_empty());
20013        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20014        assert_eq!(e2e[0].output_tail, "initial");
20015        let gate = runner.run_gate(&winner).await.unwrap();
20016        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20017        // An interrupted repair must recheck e2e from saved state. Landing
20018        // reuses that state and must run both pre-gate and gate again.
20019        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20020            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20021            passed: false,
20022            outcomes: Vec::new(),
20023        });
20024        runner.state.save().unwrap();
20025        let mut resumed = RunState::load(&runner.state.id).unwrap();
20026        let wait = crate::repo_lock::WaitPolicy::default();
20027        assert!(matches!(
20028            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20029                .await
20030                .unwrap(),
20031            GateFix::Retry
20032        ));
20033        let saved = RunState::load(&resumed.id).unwrap();
20034        let repair = saved.gate_repair_e2e.unwrap();
20035        assert!(repair.passed);
20036        assert_eq!(repair.outcomes.len(), 1);
20037        assert!(repair.outcomes[0].ok());
20038        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20039            .await
20040            .unwrap();
20041        assert_eq!(landing.len(), 1);
20042        assert!(landing[0].ok(), "{landing:?}");
20043        assert_eq!(
20044            std::fs::read_to_string(repo.join("phases")).unwrap(),
20045            "pre\ngate\ne2e\npre\ngate\n"
20046        );
20047        assert_worker();
20048    }
20049
20050    #[cfg(unix)]
20051    #[tokio::test]
20052    async fn verification_keeps_the_panic_and_the_full_output() {
20053        crate::run::pin_test_home();
20054        let tmp = tempfile::tempdir().unwrap();
20055        let mut config = Config::default();
20056        config.disk.min_free_bytes = 0;
20057        let mut state = RunState::new(
20058            tmp.path().to_owned(),
20059            "main".into(),
20060            "base".into(),
20061            "verification evidence".into(),
20062            config,
20063        );
20064        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20065                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20066                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20067        let mut names = Vec::new();
20068        for _ in 0..2 {
20069            let (outcomes, _) = run_commands(
20070                &mut state,
20071                "gate",
20072                "gate",
20073                0,
20074                &["sh".into(), "-c".into()],
20075                &[script.into()],
20076                tmp.path(),
20077                Duration::from_secs(60),
20078            )
20079            .await;
20080            let tail = &outcomes[0].output_tail;
20081            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20082            assert!(tail.contains("boom assertion"));
20083            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20084            let line = tail.lines().next().unwrap();
20085            names.push(line.to_owned());
20086        }
20087        assert_ne!(names[0], names[1], "the second run must not overwrite");
20088        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20089        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20090        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20091
20092        // Storage failures are evidence too; preserve the command's verdict.
20093        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20094        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20095        let (outcomes, _) = run_commands(
20096            &mut state,
20097            "gate",
20098            "gate",
20099            0,
20100            &["sh".into(), "-c".into()],
20101            &["echo failure-output; exit 7".into()],
20102            tmp.path(),
20103            Duration::from_secs(10),
20104        )
20105        .await;
20106        assert_eq!(outcomes[0].code, Some(7));
20107        assert!(state.events.iter().any(|e| {
20108            e.node == "gate" && e.message.contains("could not save the full gate output:")
20109        }));
20110    }
20111
20112    #[cfg(unix)]
20113    #[tokio::test]
20114    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20115        crate::run::pin_test_home();
20116        let tmp = tempfile::tempdir().unwrap();
20117        let mut config = Config::default();
20118        config.disk.min_free_bytes = 0;
20119        let mut state = RunState::new(
20120            tmp.path().to_owned(),
20121            "main".into(),
20122            "base".into(),
20123            "verification evidence".into(),
20124            config,
20125        );
20126        let (outcomes, pids) = run_commands(
20127            &mut state,
20128            "gate",
20129            "gate",
20130            1,
20131            &["sh".into(), "-c".into()],
20132            &[
20133                "printf before-timeout; exec sleep 30".into(),
20134                "kill -TERM $$".into(),
20135                "exit 7".into(),
20136                "printf success".into(),
20137            ],
20138            tmp.path(),
20139            Duration::from_secs(1),
20140        )
20141        .await;
20142        wait_for_timed_out_children_to_die(&pids).await;
20143        assert_eq!(
20144            pids.len(),
20145            1,
20146            "unleased timeout retains its death-check PID"
20147        );
20148        assert_eq!(outcomes.len(), 4);
20149        assert_eq!(outcomes[0].code, None);
20150        assert!(outcomes[0].output_tail.contains("before-timeout"));
20151        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20152        assert_eq!(outcomes[1].code, None);
20153        assert!(outcomes[1].output_tail.contains("command terminated:"));
20154        assert!(!outcomes[1].output_tail.contains("timed out"));
20155        assert_eq!(outcomes[2].code, Some(7));
20156        assert!(!outcomes[2].output_tail.contains("timed out"));
20157        assert_eq!(outcomes[3].code, Some(0));
20158        assert_eq!(outcomes[3].output_tail, "success");
20159        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20160    }
20161
20162    /// The addendum's second gap: a `verify.gate` command running for real
20163    /// wall-clock time had nothing at all to show for it in `active` before
20164    /// `run_commands` learned to record it — a run could sit in `Gating` for
20165    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20166    /// what was actually happening. Proven with a genuinely still-running
20167    /// command, not just a before/after check on the final state: a poller
20168    /// task reads the same `run.json` `gate()` is writing, the same way the
20169    /// phone or `magi show` would, while the shell command is still blocked
20170    /// on its own release marker.
20171    #[tokio::test]
20172    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20173        crate::run::pin_test_home();
20174
20175        let tmp = tempfile::tempdir().expect("tempdir");
20176        let repo = tmp.path().join("repo");
20177        std::fs::create_dir_all(&repo).unwrap();
20178        init_repo(&repo);
20179
20180        let mut config = Config::default();
20181        config.disk.min_free_bytes = 0;
20182        config.verify.gate = vec![
20183            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20184             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20185                .to_owned(),
20186        ];
20187
20188        let mut state = RunState::new(
20189            repo.clone(),
20190            "main".to_owned(),
20191            "deadbeef".to_owned(),
20192            "task".to_owned(),
20193            config,
20194        );
20195        let run_id = state.id.clone();
20196        state.candidates = vec![Candidate {
20197            index: 0,
20198            label: 'A',
20199            agent: "alpha".to_owned(),
20200            branch: "does-not-exist".to_owned(),
20201            worktree: repo.clone(),
20202            summary: String::new(),
20203            stat: String::new(),
20204            files: 0,
20205            commits: 0,
20206            empty: false,
20207            failed: None,
20208            verified_noop: None,
20209            duration_ms: 0,
20210            folded: false,
20211        }];
20212        state.tally = Some(Tally {
20213            first_choice: BTreeMap::from([('A', 1)]),
20214            borda: BTreeMap::new(),
20215            winner: 'A',
20216            rankings: 1,
20217            unanimous_initial: true,
20218            deliberated: false,
20219            changed_votes: 0,
20220            unanimous_final: true,
20221            tie_break: None,
20222            judges: 0,
20223            present: 0,
20224            quorum: 0,
20225            met_quorum: true,
20226            uncontested: Some("only candidate A produced a change".to_owned()),
20227        });
20228        state.reviews = vec![ReviewRound {
20229            round: 1,
20230            head: "deadbeef".to_owned(),
20231            verified_head: None,
20232            verified_at: None,
20233            reviews: Vec::new(),
20234            e2e: Vec::new(),
20235            fix: None,
20236            blocking: 0,
20237            answered: 0,
20238            expected: 0,
20239            clean: true,
20240            verify_retried: false,
20241            e2e_deferred: false,
20242            e2e_defer_reason: None,
20243            progressed: false,
20244            vote_split: false,
20245            reconsideration: Vec::new(),
20246            verdict: None,
20247        }];
20248
20249        let mut runner = Runner {
20250            state,
20251            roles: ResolvedRoles {
20252                implementers: Vec::new(),
20253                judges: Vec::new(),
20254                reviewers: Vec::new(),
20255                fixer: None,
20256                conductor: conductor(),
20257                implementer_roster: Vec::new(),
20258                judge_roster: Vec::new(),
20259                reviewer_roster: Vec::new(),
20260            },
20261            sem: Arc::new(Semaphore::new(1)),
20262            pause: Pause::new(),
20263            interrupt: Pause::new(),
20264        };
20265
20266        let started_marker = repo.join("started.marker");
20267        let release_marker = repo.join("release.marker");
20268        let poller = tokio::spawn(async move {
20269            // Bounded so a regression that never records the task entry
20270            // fails this test in seconds instead of hanging the suite —
20271            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20272            // does_not_cut_it_short` uses for the same reason.
20273            for _ in 0..100 {
20274                if started_marker.exists()
20275                    && let Ok(s) = crate::run::RunState::load(&run_id)
20276                    && let Some(a) = s.active.get("gate")
20277                {
20278                    std::fs::write(&release_marker, b"go").expect("release marker");
20279                    return Some(a.clone());
20280                }
20281                tokio::time::sleep(Duration::from_millis(50)).await;
20282            }
20283            None
20284        });
20285
20286        runner.gate().await.expect("gate");
20287        let captured = poller.await.expect("poller task");
20288        let captured = captured.expect(
20289            "the poller never saw a `gate` task entry in run.json while the command was \
20290             still blocked on its own release marker",
20291        );
20292
20293        assert_eq!(captured.task.as_deref(), Some("gate"));
20294        assert_eq!(captured.node, "gate");
20295        assert_eq!(captured.index, Some(1));
20296        assert_eq!(captured.total, Some(1));
20297        assert!(
20298            captured
20299                .command
20300                .as_deref()
20301                .is_some_and(|c| c.contains("started.marker")),
20302            "{captured:?}"
20303        );
20304
20305        assert!(
20306            runner.state.active.is_empty(),
20307            "the entry must be cleared once the command actually finished: {:?}",
20308            runner.state.active
20309        );
20310        assert!(runner.state.gate_ran);
20311        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20312    }
20313
20314    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20315    /// record for `land`; one with only a Minor, or no reject, leaves none.
20316    #[tokio::test]
20317    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20318        use crate::verdict::{Finding, ReviewVote, Severity};
20319        crate::run::pin_test_home();
20320        let tmp = tempfile::tempdir().expect("tempdir");
20321        let repo = tmp.path().join("repo");
20322        std::fs::create_dir_all(&repo).unwrap();
20323        init_repo(&repo);
20324
20325        for (severity, vote, expect) in [
20326            (Severity::Major, ReviewVote::Reject, true),
20327            (Severity::Minor, ReviewVote::Reject, false),
20328            (Severity::Major, ReviewVote::Approve, false),
20329        ] {
20330            let mut round = review_round(false, 1, 1, 1, false, true);
20331            round.reviews = vec![ReviewRecord {
20332                reviewer: 1,
20333                agent: "alpha".to_owned(),
20334                summary: String::new(),
20335                findings: vec![Finding {
20336                    id: "R1-1-1".to_owned(),
20337                    severity,
20338                    file: None,
20339                    line: None,
20340                    title: "t".to_owned(),
20341                    detail: String::new(),
20342                }],
20343                vote: Some(vote),
20344                failed: None,
20345                duration_ms: 0,
20346                attempts: 0,
20347            }];
20348            let mut state = RunState::new(
20349                repo.clone(),
20350                "main".to_owned(),
20351                "deadbeef".to_owned(),
20352                "task".to_owned(),
20353                Config::default(),
20354            );
20355            state.reviews = vec![round];
20356            let mut runner = Runner {
20357                state,
20358                roles: ResolvedRoles {
20359                    implementers: Vec::new(),
20360                    judges: Vec::new(),
20361                    reviewers: Vec::new(),
20362                    fixer: None,
20363                    conductor: conductor(),
20364                    implementer_roster: Vec::new(),
20365                    judge_roster: Vec::new(),
20366                    reviewer_roster: Vec::new(),
20367                },
20368                sem: Arc::new(Semaphore::new(1)),
20369                pause: Pause::new(),
20370                interrupt: Pause::new(),
20371            };
20372            let shell = runner.state.config.shell();
20373            runner
20374                .stop_reviewing("round budget spent", &shell, &repo)
20375                .await
20376                .expect("stop_reviewing");
20377            assert_eq!(runner.state.status, RunStatus::Gating);
20378            assert_eq!(
20379                runner.state.contested_handoff.is_some(),
20380                expect,
20381                "{severity:?} + {vote:?}"
20382            );
20383        }
20384    }
20385
20386    /// The shape the incident this whole fix responds to actually had: the
20387    /// round budget spent, the last round's own e2e blocked on the shared
20388    /// build cache (held here by a live pid — this test process — exactly
20389    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20390    /// without forking a process). `stop_reviewing` must retry it — not
20391    /// silently leave the round looking untouched (the catch-up-only half of
20392    /// the bug), and not read the contention as a red `e2e` and block the
20393    /// run on it (the other half). Called directly, the same way
20394    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20395    /// above exercises `gate`, so this never needs a real cargo build to
20396    /// reach: the lease is never released, so `with_cache_lease` never gets
20397    /// past acquiring it into anything that would need a real workspace.
20398    #[tokio::test]
20399    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20400        crate::run::pin_test_home();
20401        let home = crate::run::home();
20402
20403        let tmp = tempfile::tempdir().expect("tempdir");
20404        let repo = tmp.path().join("repo");
20405        std::fs::create_dir_all(&repo).unwrap();
20406        init_repo(&repo);
20407        let head = crate::git::rev_parse(&repo, "HEAD")
20408            .await
20409            .expect("rev-parse");
20410        // Unique to this test, so holding its lease cannot collide with
20411        // another test sharing the same process-wide `home`.
20412        let cache_dir = tmp.path().join("target");
20413
20414        let mut config = Config::default();
20415        config.verify.e2e = vec![format!(
20416            "CARGO_TARGET_DIR='{}' test -f README.md",
20417            cache_dir.display()
20418        )];
20419        config.graph.review_rounds = 1;
20420        // Bounded so a regression that does start waiting fails the test in
20421        // seconds, not hangs it.
20422        config.graph.timeout_verify = Some(2);
20423        config.graph.timeout_resource_wait = 2;
20424
20425        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20426        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20427            .expect("no io error acquiring directly")
20428        {
20429            crate::cache::AcquireOutcome::Acquired(g) => g,
20430            crate::cache::AcquireOutcome::Busy(b) => {
20431                panic!("expected the direct acquire to win the lease first: {b:?}")
20432            }
20433        };
20434
20435        let mut state = RunState::new(
20436            repo.clone(),
20437            "main".to_owned(),
20438            head.clone(),
20439            "task".to_owned(),
20440            config,
20441        );
20442        state.candidates = vec![Candidate {
20443            index: 0,
20444            label: 'A',
20445            agent: "alpha".to_owned(),
20446            branch: "does-not-exist".to_owned(),
20447            worktree: repo.clone(),
20448            summary: String::new(),
20449            stat: String::new(),
20450            files: 0,
20451            commits: 0,
20452            empty: false,
20453            failed: None,
20454            verified_noop: None,
20455            duration_ms: 0,
20456            folded: false,
20457        }];
20458        state.tally = Some(Tally {
20459            first_choice: BTreeMap::from([('A', 1)]),
20460            borda: BTreeMap::new(),
20461            winner: 'A',
20462            rankings: 1,
20463            unanimous_initial: true,
20464            deliberated: false,
20465            changed_votes: 0,
20466            unanimous_final: true,
20467            tie_break: None,
20468            judges: 0,
20469            present: 0,
20470            quorum: 0,
20471            met_quorum: true,
20472            uncontested: Some("only candidate A produced a change".to_owned()),
20473        });
20474        // The round budget's last round, deferred: `needs_catchup_run`'s
20475        // other trigger. `stop_reviewing`'s retry machinery must treat this
20476        // exactly like a resource-blocked attempt once it actually runs.
20477        state.reviews = vec![ReviewRound {
20478            round: 1,
20479            head: head.clone(),
20480            verified_head: None,
20481            verified_at: None,
20482            reviews: Vec::new(),
20483            e2e: Vec::new(),
20484            fix: None,
20485            blocking: 1,
20486            answered: 1,
20487            expected: 1,
20488            clean: false,
20489            verify_retried: false,
20490            e2e_deferred: true,
20491            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20492            progressed: false,
20493            vote_split: false,
20494            reconsideration: Vec::new(),
20495            verdict: None,
20496        }];
20497
20498        let mut runner = Runner {
20499            state,
20500            roles: ResolvedRoles {
20501                implementers: Vec::new(),
20502                judges: Vec::new(),
20503                reviewers: Vec::new(),
20504                fixer: None,
20505                conductor: conductor(),
20506                implementer_roster: Vec::new(),
20507                judge_roster: Vec::new(),
20508                reviewer_roster: Vec::new(),
20509            },
20510            sem: Arc::new(Semaphore::new(1)),
20511            pause: Pause::new(),
20512            interrupt: Pause::new(),
20513        };
20514
20515        let shell = runner.state.config.shell();
20516        runner
20517            .stop_reviewing("round budget spent", &shell, &repo)
20518            .await
20519            .expect("stop_reviewing");
20520
20521        let last = runner.state.reviews.last().expect("round record");
20522        assert_eq!(
20523            last.e2e_status(),
20524            E2eStatus::ResourceBlocked,
20525            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20526             failed: {last:?}"
20527        );
20528        assert_eq!(
20529            last.verified_head.as_deref(),
20530            Some(head.as_str()),
20531            "which commit this attempt targeted is known even though nothing finished checking \
20532             it"
20533        );
20534        let first_attempt_at = last
20535            .verified_at
20536            .expect("when this attempt ran is known too");
20537        assert_ne!(
20538            runner.state.status,
20539            RunStatus::Blocked,
20540            "contention is evidence about the machine, not the patch — it must not settle the \
20541             run as blocked: {:?}",
20542            runner.state.status
20543        );
20544        assert!(
20545            !runner
20546                .state
20547                .events
20548                .iter()
20549                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20550            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20551            runner.state.events
20552        );
20553
20554        // The cache is still held: a later reentry must retry the same
20555        // round's verification again — not leave it looking exactly as
20556        // untouched as the first blocked attempt, which is indistinguishable
20557        // from never having tried again at all.
20558        runner
20559            .stop_reviewing("round budget spent", &shell, &repo)
20560            .await
20561            .expect("stop_reviewing retry");
20562        assert_eq!(
20563            runner.state.reviews.len(),
20564            1,
20565            "no new round was started: {:?}",
20566            runner.state.reviews
20567        );
20568        let last = runner.state.reviews.last().expect("round record");
20569        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20570        assert!(
20571            last.verified_at.expect("still known") > first_attempt_at,
20572            "a second reentry must be a fresh attempt, not a stale copy of the first"
20573        );
20574        assert_ne!(runner.state.status, RunStatus::Blocked);
20575
20576        held.release();
20577    }
20578
20579    #[cfg(unix)]
20580    #[tokio::test]
20581    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20582        ask_test_home();
20583        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20584            let tmp = tempfile::tempdir().unwrap();
20585            let repo = tmp.path().join("repo");
20586            std::fs::create_dir(&repo).unwrap();
20587            init_repo(&repo);
20588            let run_git = |args: &[&str]| {
20589                let out = std::process::Command::new("git")
20590                    .args(args)
20591                    .current_dir(&repo)
20592                    .output()
20593                    .unwrap();
20594                assert!(
20595                    out.status.success(),
20596                    "{args:?}: {}",
20597                    String::from_utf8_lossy(&out.stderr)
20598                );
20599            };
20600            let origin = tmp.path().join("origin.git");
20601            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20602            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20603            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20604            run_git(&["add", "spec.rs"]);
20605            run_git(&["commit", "-m", "initial spec"]);
20606            run_git(&["push", "origin", "main"]);
20607            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20608            run_git(&["checkout", "-b", "fix/review"]);
20609            std::fs::write(
20610                repo.join("check.rs"),
20611                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20612            )
20613            .unwrap();
20614            run_git(&["add", "check.rs"]);
20615            run_git(&["commit", "-m", "use spec"]);
20616            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20617            run_git(&["checkout", "main"]);
20618            std::fs::write(
20619                repo.join("spec.rs"),
20620                "struct AgentSpec { max_turn_secs: u64 }\n",
20621            )
20622            .unwrap();
20623            run_git(&["add", "spec.rs"]);
20624            run_git(&["commit", "-m", "extend spec"]);
20625            run_git(&["push", "origin", "main"]);
20626            run_git(&["checkout", "fix/review"]);
20627
20628            let mut runner = runner_at(RunStatus::Reviewing);
20629            runner.state.repo = repo.clone();
20630            runner.state.base_commit = base;
20631            runner.state.pr_head = Some("fix/review".into());
20632            runner.state.config.graph.review_rounds = max_rounds;
20633            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20634            runner.state.config.graph.sessions = false;
20635            runner.state.config.disk.min_free_bytes = 0;
20636            runner.state.config.verify.e2e =
20637                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20638            runner.state.candidates = vec![Candidate {
20639                index: 0,
20640                label: 'A',
20641                agent: "fixture".into(),
20642                branch: "fix/review".into(),
20643                worktree: repo.clone(),
20644                summary: String::new(),
20645                stat: String::new(),
20646                files: 1,
20647                commits: 1,
20648                empty: false,
20649                failed: None,
20650                verified_noop: None,
20651                duration_ms: 0,
20652                folded: false,
20653            }];
20654            runner.state.tally = Some(Tally {
20655                first_choice: BTreeMap::new(),
20656                borda: BTreeMap::new(),
20657                winner: 'A',
20658                rankings: 0,
20659                unanimous_initial: true,
20660                deliberated: false,
20661                changed_votes: 0,
20662                unanimous_final: true,
20663                tie_break: None,
20664                judges: 0,
20665                present: 0,
20666                quorum: 0,
20667                met_quorum: true,
20668                uncontested: Some("review-only".into()),
20669            });
20670            let mut last = review_round(false, 0, 1, 1, false, false);
20671            last.head = reviewed.clone();
20672            last.e2e_deferred = true;
20673            last.e2e[0].resource_blocked = true;
20674            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20675            if max_rounds == 1 {
20676                last.e2e.clear();
20677            }
20678            runner.state.reviews = vec![last];
20679            let calls = tmp.path().join("fix-calls");
20680            let mut fixer = spec("repair");
20681            fixer
20682                .env
20683                .insert("FIX_CALLS".into(), calls.display().to_string());
20684            fixer.command = vec!["sh".into(), "-c".into(), r#"
20685                echo fix >> "$FIX_CALLS"
20686                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20687                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20688                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20689                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20690            "#.into()];
20691            let mut reviewer = spec("reviewer");
20692            reviewer.command = vec![
20693                "sh".into(),
20694                "-c".into(),
20695                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20696                    .into(),
20697            ];
20698            runner.roles.fixer = Some(vec![fixer]);
20699            runner.roles.reviewers = vec![reviewer];
20700
20701            runner.sync_to_base().await.unwrap();
20702            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20703            assert_ne!(rebased, reviewed);
20704            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20705            if reentry {
20706                // Simulate a crash after catch-up evidence was saved but before
20707                // dispatching the fixer. No dedicated pending-repair field exists.
20708                let shell = runner.state.config.shell();
20709                runner
20710                    .stop_unresolved_review("deferred verification", &shell, &repo)
20711                    .await
20712                    .unwrap();
20713                assert!(runner.catchup_repairable());
20714                assert!(!calls.exists());
20715                runner.state = RunState::load(&runner.state.id).unwrap();
20716                assert!(runner.catchup_repairable());
20717            }
20718            runner.review_loop().await.unwrap();
20719            let first = &runner.state.reviews[0];
20720            assert_eq!(first.head, reviewed);
20721            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20722            assert!(
20723                first.verify_retried,
20724                "a reproduced build failure gets exactly one retry: {:?}",
20725                first.e2e
20726            );
20727            assert!(first.e2e[0].output_tail.contains("E0063"));
20728            assert!(!runner.catchup_repairable());
20729            if max_rounds == 2 {
20730                assert_eq!(
20731                    runner.state.status,
20732                    RunStatus::Gating,
20733                    "{}",
20734                    crate::report::run(&runner.state)
20735                );
20736                assert_eq!(runner.state.reviews.len(), 2);
20737                assert!(first.fix.as_ref().unwrap().committed);
20738                assert!(runner.state.reviews[1].clean);
20739                assert_eq!(
20740                    runner
20741                        .state
20742                        .reviews
20743                        .iter()
20744                        .filter(|r| r.fix.is_some())
20745                        .count(),
20746                    1
20747                );
20748                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20749                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20750            } else {
20751                assert_eq!(runner.state.status, RunStatus::Blocked);
20752                assert!(first.fix.is_none());
20753                assert!(!calls.exists());
20754                let recovery = crate::recovery::assess(
20755                    "prior",
20756                    Some(&runner.state),
20757                    &runner.landing_base(),
20758                    crate::recovery::Relation::Diverged,
20759                );
20760                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20761                let reasons = recovery.reasons.join("\n");
20762                assert!(
20763                    reasons.contains(&reviewed) && reasons.contains(&rebased),
20764                    "{reasons}"
20765                );
20766                assert!(
20767                    reasons.contains("base sync to")
20768                        && reasons.contains("additional review is required"),
20769                    "{reasons}"
20770                );
20771            }
20772        }
20773    }
20774
20775    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
20776    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
20777    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
20778    /// own top-of-function fast path (`review_conclusion`) correctly reads
20779    /// this shape as `None` rather than guessing `Blocked`, and the loop's
20780    /// own `for` range is empty once the round budget is spent, so
20781    /// `review_loop` must retry the check itself rather than silently doing
20782    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
20783    /// above exercises directly, but through `review_loop`'s own entry point
20784    /// this time, proving the wiring between the two rather than just the
20785    /// retry logic in isolation.
20786    #[tokio::test]
20787    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20788        crate::run::pin_test_home();
20789        let home = crate::run::home();
20790
20791        let tmp = tempfile::tempdir().expect("tempdir");
20792        let repo = tmp.path().join("repo");
20793        std::fs::create_dir_all(&repo).unwrap();
20794        init_repo(&repo);
20795        let head = crate::git::rev_parse(&repo, "HEAD")
20796            .await
20797            .expect("rev-parse");
20798        let cache_dir = tmp.path().join("target");
20799
20800        let mut config = Config::default();
20801        config.verify.e2e = vec![format!(
20802            "CARGO_TARGET_DIR='{}' test -f README.md",
20803            cache_dir.display()
20804        )];
20805        config.graph.review_rounds = 1;
20806        config.graph.timeout_verify = Some(2);
20807        config.graph.timeout_resource_wait = 2;
20808
20809        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20810        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20811            .expect("no io error acquiring directly")
20812        {
20813            crate::cache::AcquireOutcome::Acquired(g) => g,
20814            crate::cache::AcquireOutcome::Busy(b) => {
20815                panic!("expected the direct acquire to win the lease first: {b:?}")
20816            }
20817        };
20818
20819        let mut state = RunState::new(
20820            repo.clone(),
20821            "main".to_owned(),
20822            head.clone(),
20823            "task".to_owned(),
20824            config,
20825        );
20826        state.candidates = vec![Candidate {
20827            index: 0,
20828            label: 'A',
20829            agent: "alpha".to_owned(),
20830            branch: "does-not-exist".to_owned(),
20831            worktree: repo.clone(),
20832            summary: String::new(),
20833            stat: String::new(),
20834            files: 0,
20835            commits: 0,
20836            empty: false,
20837            failed: None,
20838            verified_noop: None,
20839            duration_ms: 0,
20840            folded: false,
20841        }];
20842        state.tally = Some(Tally {
20843            first_choice: BTreeMap::from([('A', 1)]),
20844            borda: BTreeMap::new(),
20845            winner: 'A',
20846            rankings: 1,
20847            unanimous_initial: true,
20848            deliberated: false,
20849            changed_votes: 0,
20850            unanimous_final: true,
20851            tie_break: None,
20852            judges: 0,
20853            present: 0,
20854            quorum: 0,
20855            met_quorum: true,
20856            uncontested: Some("only candidate A produced a change".to_owned()),
20857        });
20858        // The exact shape a prior process's `stop_reviewing` would have left
20859        // on disk: the round budget's last round, a real attempt already
20860        // made and already resource-blocked.
20861        state.reviews = vec![ReviewRound {
20862            round: 1,
20863            head: head.clone(),
20864            verified_head: Some(head.clone()),
20865            verified_at: Some(jiff::Timestamp::now()),
20866            reviews: Vec::new(),
20867            e2e: vec![CommandOutcome {
20868                command: format!(
20869                    "CARGO_TARGET_DIR='{}' test -f README.md",
20870                    cache_dir.display()
20871                ),
20872                code: None,
20873                output_tail: "waiting for the shared build cache".to_owned(),
20874                duration_ms: 0,
20875                resource_blocked: true,
20876            }],
20877            fix: None,
20878            blocking: 1,
20879            answered: 1,
20880            expected: 1,
20881            clean: false,
20882            verify_retried: false,
20883            e2e_deferred: false,
20884            e2e_defer_reason: None,
20885            progressed: false,
20886            vote_split: false,
20887            reconsideration: Vec::new(),
20888            verdict: None,
20889        }];
20890
20891        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20892        let mut runner = Runner {
20893            state,
20894            roles: ResolvedRoles {
20895                implementers: Vec::new(),
20896                judges: Vec::new(),
20897                reviewers: Vec::new(),
20898                fixer: None,
20899                conductor: conductor(),
20900                implementer_roster: Vec::new(),
20901                judge_roster: Vec::new(),
20902                reviewer_roster: Vec::new(),
20903            },
20904            sem: Arc::new(Semaphore::new(1)),
20905            pause: Pause::new(),
20906            interrupt: Pause::new(),
20907        };
20908
20909        // The lease is still held throughout, so this reentry's own retry is
20910        // also contended — proving `review_loop` actually tried again (not
20911        // that it happened to succeed) is what the timestamp comparison
20912        // below is for.
20913        runner.review_loop().await.expect("review_loop");
20914
20915        assert_eq!(
20916            runner.state.reviews.len(),
20917            1,
20918            "no new round was started on top of the unresolved one: {:?}",
20919            runner.state.reviews
20920        );
20921        let last = &runner.state.reviews[0];
20922        assert_eq!(
20923            last.e2e_status(),
20924            E2eStatus::ResourceBlocked,
20925            "still contended: {last:?}"
20926        );
20927        assert!(
20928            last.verified_at.expect("still known") > first_attempt_at,
20929            "review_loop must have actually retried the check, not left it exactly as found"
20930        );
20931        assert_ne!(
20932            runner.state.status,
20933            RunStatus::Blocked,
20934            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20935            runner.state.status
20936        );
20937
20938        held.release();
20939    }
20940
20941    #[tokio::test]
20942    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20943        crate::run::pin_test_home();
20944        for status in [RunStatus::Blocked, RunStatus::Gating] {
20945            let mut runner = runner_at(status);
20946            runner.state.config.graph.land = true;
20947            runner.state.gate_ran = true;
20948            runner.state.merge = Some(MergeOutcome {
20949                mode: MergeMode::Pr,
20950                ok: false,
20951                detail: "the fixer produced no commit".into(),
20952                empty: false,
20953            });
20954            let url = "https://example.invalid/o/r/pull/1";
20955            runner.state.pr = Some(crate::run::PrRecord {
20956                url: url.into(),
20957                number: 1,
20958                state: "open".into(),
20959                checks: "red".into(),
20960                round: 1,
20961                rounds: 1,
20962                red_at_merge: Vec::new(),
20963            });
20964            runner.state.save().unwrap();
20965            // The nonexistent repository makes observation fail before a forge
20966            // command can run. We still must reach land, rather than silently
20967            // return from the graph with its old gate and merge records.
20968            runner.execute_graph().await.unwrap();
20969            assert_eq!(runner.state.status, RunStatus::Blocked);
20970            assert!(runner.state.land_sync_pending);
20971            assert!(runner.state.events.iter().any(|event|
20972                event.node == "land" && event.message == format!("watching {url}")));
20973            assert_eq!(
20974                runner.state.merge.as_ref().unwrap().detail,
20975                "the fixer produced no commit"
20976            );
20977        }
20978    }
20979
20980    #[tokio::test]
20981    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
20982        crate::run::pin_test_home();
20983        let tmp = tempfile::tempdir().expect("tempdir");
20984        let repo = tmp.path().join("repo");
20985        std::fs::create_dir_all(&repo).unwrap();
20986        init_repo(&repo);
20987
20988        let mut config = Config::default();
20989        config.merge.mode = MergeMode::Pr;
20990        config.graph.land = true;
20991        config.graph.land_approval = false;
20992
20993        let mut state = RunState::new(
20994            repo.clone(),
20995            "main".to_owned(),
20996            "deadbeef".to_owned(),
20997            "task".to_owned(),
20998            config,
20999        );
21000        state.candidates = vec![Candidate {
21001            index: 0,
21002            label: 'A',
21003            agent: "alpha".to_owned(),
21004            branch: "does-not-exist".to_owned(),
21005            worktree: repo.clone(),
21006            summary: String::new(),
21007            stat: String::new(),
21008            files: 0,
21009            commits: 0,
21010            empty: false,
21011            failed: None,
21012            verified_noop: None,
21013            duration_ms: 0,
21014            folded: false,
21015        }];
21016        state.tally = Some(Tally {
21017            first_choice: BTreeMap::from([('A', 1)]),
21018            borda: BTreeMap::new(),
21019            winner: 'A',
21020            rankings: 1,
21021            unanimous_initial: true,
21022            deliberated: false,
21023            changed_votes: 0,
21024            unanimous_final: true,
21025            tie_break: None,
21026            judges: 0,
21027            present: 0,
21028            quorum: 0,
21029            met_quorum: true,
21030            uncontested: Some("only candidate A produced a change".to_owned()),
21031        });
21032        state.reviews = vec![ReviewRound {
21033            round: 1,
21034            head: "deadbeef".to_owned(),
21035            verified_head: None,
21036            verified_at: None,
21037            reviews: Vec::new(),
21038            e2e: Vec::new(),
21039            fix: None,
21040            blocking: 0,
21041            answered: 0,
21042            expected: 0,
21043            clean: true,
21044            verify_retried: false,
21045            e2e_deferred: false,
21046            e2e_defer_reason: None,
21047            progressed: false,
21048            vote_split: false,
21049            reconsideration: Vec::new(),
21050            verdict: None,
21051        }];
21052        state.gate = vec![CommandOutcome {
21053            command: "test".to_owned(),
21054            code: Some(0),
21055            output_tail: String::new(),
21056            duration_ms: 0,
21057            resource_blocked: false,
21058        }];
21059        state.gate_ran = true;
21060        // A first pass through `merge` already pushed and opened this pull
21061        // request; `status` is `Landing` because a previous call into `land`
21062        // parked or was interrupted before it reached a terminal outcome.
21063        state.status = RunStatus::Landing;
21064        state.merge = Some(MergeOutcome {
21065            mode: MergeMode::Pr,
21066            ok: true,
21067            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21068            empty: false,
21069        });
21070
21071        // The Landing-resume shortcut calls `run_land` directly rather than
21072        // through `merge`, which is exactly the call site that used to skip
21073        // `settle_questions` - see the fixture below.
21074        ask_test_home();
21075        // A real resume loads a persisted run; driver ownership is checked
21076        // against that record before the Landing shortcut is entered.
21077        state.save().expect("save landing run");
21078        let store = ask::Questions::open();
21079        let q = ask_open_question(&store, &state.id);
21080
21081        let mut runner = Runner {
21082            state,
21083            roles: ResolvedRoles {
21084                implementers: Vec::new(),
21085                judges: Vec::new(),
21086                reviewers: Vec::new(),
21087                fixer: None,
21088                conductor: conductor(),
21089                implementer_roster: Vec::new(),
21090                judge_roster: Vec::new(),
21091                reviewer_roster: Vec::new(),
21092            },
21093            sem: Arc::new(Semaphore::new(1)),
21094            pause: Pause::new(),
21095            interrupt: Pause::new(),
21096        };
21097
21098        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21099        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21100        // exactly because `review_loop` would otherwise clobber the marker
21101        // first.
21102        runner.execute().await.expect("execute");
21103
21104        assert_eq!(
21105            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21106            Some("https://example.invalid/x/y/pull/1"),
21107            "reentry must not push again or open a second pull request over the \
21108             one `land` is already watching"
21109        );
21110        assert_ne!(
21111            runner.state.status,
21112            RunStatus::Landing,
21113            "land could not actually reach the fake pull request, so it must \
21114             have given up rather than left the run silently parked forever"
21115        );
21116        // `land` could not reach the fake pull request, so it gave up into
21117        // `Blocked` - still resumable, so the question must not have been
21118        // swept just because this branch now also calls `settle_questions`.
21119        assert_eq!(runner.state.status, RunStatus::Blocked);
21120        assert!(
21121            store.get(&q.id).unwrap().status.open(),
21122            "Blocked is still alive; settle_questions must have been a no-op here"
21123        );
21124    }
21125
21126    fn state_with_round(round: ReviewRound) -> RunState {
21127        let mut s = RunState::new(
21128            PathBuf::from("/repo"),
21129            "main".to_owned(),
21130            "abc1234".to_owned(),
21131            "add retries".to_owned(),
21132            Config::default(),
21133        );
21134        s.reviews = vec![round];
21135        s
21136    }
21137
21138    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21139        crate::verdict::Finding {
21140            id: id.to_owned(),
21141            severity,
21142            file: None,
21143            line: None,
21144            title: title.to_owned(),
21145            detail: String::new(),
21146        }
21147    }
21148
21149    #[test]
21150    fn pr_body_names_open_findings_and_declined_ones() {
21151        let round = ReviewRound {
21152            round: 2,
21153            head: "deadbee".to_owned(),
21154            verified_head: None,
21155            verified_at: None,
21156            reviews: vec![ReviewRecord {
21157                attempts: 0,
21158                reviewer: 1,
21159                agent: "alpha".to_owned(),
21160                summary: String::new(),
21161                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21162                vote: None,
21163                failed: None,
21164                duration_ms: 0,
21165            }],
21166            e2e: vec![CommandOutcome {
21167                command: "cargo test".to_owned(),
21168                code: Some(0),
21169                output_tail: String::new(),
21170                duration_ms: 0,
21171                resource_blocked: false,
21172            }],
21173            verify_retried: false,
21174            e2e_deferred: false,
21175            e2e_defer_reason: None,
21176            fix: Some(FixRecord {
21177                agent: "alpha".to_owned(),
21178                addressed: Vec::new(),
21179                rejected: vec![crate::verdict::Rejection {
21180                    id: "R1-1-1".to_owned(),
21181                    why: "not reachable from any caller".to_owned(),
21182                }],
21183                notes: String::new(),
21184                committed: true,
21185                failed: None,
21186                duration_ms: 0,
21187                continuation: None,
21188            }),
21189            blocking: 0,
21190            answered: 1,
21191            expected: 1,
21192            clean: false,
21193            progressed: true,
21194            vote_split: false,
21195            reconsideration: Vec::new(),
21196            verdict: None,
21197        };
21198        let state = state_with_round(round);
21199        let body = pr_message(&state, 'A').body;
21200
21201        assert!(body.contains("add retries"), "the task must still be there");
21202        assert!(body.contains("R2-1-1"), "{body}");
21203        assert!(!body.contains("unused import"), "{body}");
21204        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21205        assert!(
21206            !body.contains("not reachable from any caller"),
21207            "the reason it was declined: {body}"
21208        );
21209    }
21210
21211    #[test]
21212    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21213        let round = ReviewRound {
21214            round: 1,
21215            head: "deadbee".to_owned(),
21216            verified_head: None,
21217            verified_at: None,
21218            reviews: vec![ReviewRecord {
21219                attempts: 0,
21220                reviewer: 1,
21221                agent: "alpha".to_owned(),
21222                summary: String::new(),
21223                findings: Vec::new(),
21224                vote: None,
21225                failed: None,
21226                duration_ms: 0,
21227            }],
21228            e2e: Vec::new(),
21229            verify_retried: false,
21230            e2e_deferred: false,
21231            e2e_defer_reason: None,
21232            fix: None,
21233            blocking: 0,
21234            answered: 1,
21235            expected: 1,
21236            clean: true,
21237            progressed: false,
21238            vote_split: false,
21239            reconsideration: Vec::new(),
21240            verdict: None,
21241        };
21242        let state = state_with_round(round);
21243        let body = pr_message(&state, 'A').body;
21244        assert!(!body.contains("Open review findings"), "{body}");
21245        assert!(!body.contains("Declined"), "{body}");
21246    }
21247
21248    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21249        let mut state = RunState::new(
21250            PathBuf::from("/repo"),
21251            "main".to_owned(),
21252            "abc1234".to_owned(),
21253            instruction.to_owned(),
21254            Config::default(),
21255        );
21256        // These fixtures test execution flow, independent of host disk capacity.
21257        state.config.disk.min_free_bytes = 0;
21258        state.candidates.push(Candidate {
21259            index: 0,
21260            label: 'A',
21261            agent: "alpha".to_owned(),
21262            branch: "magi/x/A".to_owned(),
21263            worktree: PathBuf::from("/wt"),
21264            summary: summary.to_owned(),
21265            stat: String::new(),
21266            files: 1,
21267            commits: 1,
21268            empty: false,
21269            failed: None,
21270            verified_noop: None,
21271            folded: false,
21272            duration_ms: 0,
21273        });
21274        state
21275    }
21276
21277    fn review_state(subjects: &[&str]) -> RunState {
21278        let mut state = state_with_summary(
21279            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21280            "",
21281        );
21282        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21283        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21284        state
21285    }
21286
21287    /// A branch rebased onto a main that moved past the recorded
21288    /// `base_commit` is titled from its own first commit, never main's.
21289    #[tokio::test]
21290    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21291        ask_test_home();
21292        let tmp = tempfile::tempdir().unwrap();
21293        let repo = tmp.path().join("repo");
21294        std::fs::create_dir_all(&repo).unwrap();
21295        init_repo(&repo);
21296        let origin = tmp.path().join("origin.git");
21297        let g = |dir: &Path, args: &[&str]| {
21298            let out = std::process::Command::new("git")
21299                .args(args)
21300                .current_dir(dir)
21301                .quiet()
21302                .output()
21303                .expect("spawn git");
21304            assert!(
21305                out.status.success(),
21306                "git {args:?}: {}",
21307                String::from_utf8_lossy(&out.stderr)
21308            );
21309        };
21310        g(
21311            tmp.path(),
21312            &[
21313                "clone",
21314                "--bare",
21315                "-q",
21316                repo.to_str().unwrap(),
21317                origin.to_str().unwrap(),
21318            ],
21319        );
21320        g(
21321            &repo,
21322            &["remote", "add", "origin", origin.to_str().unwrap()],
21323        );
21324        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21325
21326        // Main moves on; the branch is built on top of the new main.
21327        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21328        g(&repo, &["add", "-A"]);
21329        g(
21330            &repo,
21331            &["commit", "-q", "-m", "chore(deps): update a crate"],
21332        );
21333        g(&repo, &["push", "-q", "origin", "main"]);
21334        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21335        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21336        g(&repo, &["add", "-A"]);
21337        g(
21338            &repo,
21339            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21340        );
21341        g(&repo, &["checkout", "-q", "main"]);
21342
21343        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21344        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21345        // Without a readable tracking ref the recorded base's merge base is used.
21346        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21347        assert_eq!(fallback, c1);
21348
21349        let mut state = review_state(&[
21350            "chore(deps): update a crate",
21351            "fix(daemon): apply a chosen action",
21352        ]);
21353        state.repo = repo.clone();
21354        state.base_branch = "main".to_owned();
21355        state.base_commit = c1;
21356        refresh_reviewed_commits(&mut state, "feat/own").await;
21357        assert_eq!(
21358            state.reviewed_commits,
21359            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21360        );
21361        assert_eq!(
21362            review_title(&state).as_deref(),
21363            Some("fix(daemon): apply a chosen action")
21364        );
21365        assert_eq!(
21366            leaked_subjects(&state, "feat/own").await,
21367            Some(vec!["chore(deps): update a crate".to_owned()])
21368        );
21369        // A stale tracking ref is still used when the fetch fails, but the
21370        // leak list is withheld.
21371        state.config.merge.remote = "nowhere".to_owned();
21372        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21373    }
21374
21375    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21376    /// with `origin/main` one commit behind it, and a summarizing agent that
21377    /// prints whatever `reply` holds.
21378    struct SummaryFixture {
21379        _tmp: tempfile::TempDir,
21380        repo: PathBuf,
21381        reply: PathBuf,
21382        runner: Runner,
21383    }
21384
21385    fn summary_fixture() -> SummaryFixture {
21386        ask_test_home();
21387        let tmp = tempfile::tempdir().unwrap();
21388        let repo = tmp.path().join("repo");
21389        std::fs::create_dir_all(&repo).unwrap();
21390        init_repo(&repo);
21391        let origin = tmp.path().join("origin.git");
21392        let g = |args: &[&str]| {
21393            let out = std::process::Command::new("git")
21394                .args(args)
21395                .current_dir(&repo)
21396                .quiet()
21397                .output()
21398                .expect("spawn git");
21399            assert!(
21400                out.status.success(),
21401                "git {args:?}: {}",
21402                String::from_utf8_lossy(&out.stderr)
21403            );
21404        };
21405        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21406        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21407        g(&["push", "-q", "origin", "main"]);
21408        g(&["checkout", "-q", "-b", "feat/own"]);
21409        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21410        g(&["add", "-A"]);
21411        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21412        let reply = tmp.path().join("reply.txt");
21413        std::fs::write(
21414            &reply,
21415            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21416        )
21417        .unwrap();
21418        let mut runner = runner_at(RunStatus::Gating);
21419        let mut state = review_state(&["feat(own): add the own file"]);
21420        state.repo = repo.clone();
21421        state.base_branch = "main".to_owned();
21422        state.candidates[0].branch = "feat/own".to_owned();
21423        state.candidates[0].worktree = repo.clone();
21424        let mut author = spec("alpha");
21425        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21426        author
21427            .env
21428            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21429        state.config.agents = vec![author];
21430        runner.state = state;
21431        SummaryFixture {
21432            _tmp: tmp,
21433            repo,
21434            reply,
21435            runner,
21436        }
21437    }
21438
21439    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21440        let winner = f.runner.state.candidates[0].clone();
21441        f.runner
21442            .guarded_pr_message(&winner, None, true)
21443            .await
21444            .unwrap()
21445            .expect("an English description never parks")
21446    }
21447
21448    fn clean_round_with_minor_findings() -> ReviewRound {
21449        let mut round = passing_round(2, None);
21450        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21451        round.reviews = vec![ReviewRecord {
21452            attempts: 0,
21453            reviewer: 1,
21454            agent: "alpha".into(),
21455            summary: "内部レビュー".into(),
21456            findings: vec![
21457                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21458                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21459            ],
21460            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21461            failed: None,
21462            duration_ms: 0,
21463        }];
21464        round
21465    }
21466
21467    #[tokio::test]
21468    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21469        for review_only in [false, true] {
21470            let mut f = summary_fixture();
21471            if !review_only {
21472                f.runner.state.candidates[0].agent = "alpha".into();
21473            }
21474            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21475            assert!(f.runner.state.reviews[0].clean);
21476            assert!(f.runner.state.reviews[0].fix.is_none());
21477            assert!(f.runner.state.open_findings().is_empty());
21478            assert_eq!(
21479                crate::pr_outcome::required_finding_ids(&f.runner.state),
21480                ["R2-1-1", "R2-1-2"]
21481            );
21482            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21483            let evidence = crate::pr_outcome::summary_evidence(
21484                &f.runner.state,
21485                crate::pr_outcome::FinalHead::At(&head),
21486            );
21487            assert!(
21488                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21489            );
21490            // The ordinary reply omits both IDs: no cached description is adopted.
21491            let omitted = posted(&mut f).await;
21492            assert!(omitted.body.contains("No description"));
21493            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21494            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21495            assert!(f.runner.state.change_summary.is_none());
21496            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21497            let described = posted(&mut f).await;
21498            assert!(described.body.contains("improve documentation for callers"));
21499            assert!(described.body.contains("remaining edge case"));
21500            assert!(described.body.contains("## Open review findings"));
21501            assert!(!described.body.contains("未修正"));
21502        }
21503    }
21504
21505    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21506        let mut f = summary_fixture();
21507        if !review_only {
21508            f.runner.state.candidates[0].agent = "alpha".into();
21509        }
21510        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21511        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21512        fixed
21513            .fix
21514            .as_mut()
21515            .unwrap()
21516            .rejected
21517            .push(crate::verdict::Rejection {
21518                id: "R1-1-1".into(),
21519                why: "呼び出されないコードのため辞退".into(),
21520            });
21521        let mut clean = clean_round_with_minor_findings();
21522        clean.verified_head = Some(head.clone());
21523        f.runner.state.reviews = vec![fixed, clean];
21524        ok_gate(&mut f.runner.state, Some(&head));
21525        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21526        f.runner.state.config.agents.push(scripted("beta", reply));
21527        f.runner.state.config.roles.language_judge =
21528            Some(crate::config::AgentChoice::One("beta".into()));
21529        f
21530    }
21531
21532    #[tokio::test]
21533    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21534        for review_only in [false, true] {
21535            let mut f = rewrite_fixture_with_findings(review_only,
21536                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21537            let m = posted(&mut f).await;
21538            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21539            let expected =
21540                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21541            assert!(m.body.contains(&expected));
21542            assert!(m.body.contains("## Review rounds"));
21543            assert!(
21544                m.body
21545                    .contains("Gate (`verify.gate`): passed on the final commit")
21546            );
21547            assert!(
21548                m.body
21549                    .contains("Tests (`verify.e2e`): passed on the final commit")
21550            );
21551            let c = f.runner.state.change_summary.as_ref().unwrap();
21552            assert!(
21553                m.body
21554                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21555            );
21556            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21557            assert!(crate::pr_outcome::covers_required_findings(
21558                &f.runner.state,
21559                &m.body
21560            ));
21561            assert!(m.body.contains("documentation remains incomplete"));
21562            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21563            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21564            assert!(f.runner.state.github_text.is_none());
21565        }
21566    }
21567
21568    #[tokio::test]
21569    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21570        for review_only in [false, true] {
21571            for reply in [
21572                "TITLE: fix: public description\nA short English description.",
21573                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21574            ] {
21575                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21576                let winner = f.runner.state.candidates[0].clone();
21577                let posted = f
21578                    .runner
21579                    .guarded_pr_message(&winner, None, true)
21580                    .await
21581                    .unwrap();
21582                assert!(posted.is_none());
21583                assert!(f.runner.state.parked);
21584                let withheld = f.runner.state.github_text.as_ref().unwrap();
21585                assert!(
21586                    withheld.rewrite.is_none(),
21587                    "an incomplete rewrite must not be selectable"
21588                );
21589                let q = ask::Questions::open()
21590                    .get(withheld.question.as_ref().unwrap())
21591                    .unwrap();
21592                assert!(
21593                    !q.choices
21594                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21595                );
21596                assert!(
21597                    q.detail.contains("R1-1-1")
21598                        && q.detail.contains("R2-1-1")
21599                        && q.detail.contains("R2-1-2")
21600                );
21601            }
21602        }
21603    }
21604
21605    #[tokio::test]
21606    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21607        for review_only in [false, true] {
21608            let mut f = summary_fixture();
21609            if !review_only {
21610                f.runner.state.candidates[0].agent = "alpha".into();
21611            }
21612            f.runner.state.instruction = "内部タスクの説明".into();
21613            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21614            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21615            f.runner.state.reviews = vec![
21616                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21617                round_with_notes(2, Some("二回目の内部修正報告です。")),
21618                passing_round(3, Some(&head)),
21619            ];
21620            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21621            ok_gate(&mut f.runner.state, Some(&head));
21622            let captured = f.repo.join("public-prompt.txt");
21623            let author = &mut f.runner.state.config.agents[0];
21624            author.command = vec![
21625                "sh".into(),
21626                "-c".into(),
21627                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21628            ];
21629            author.env.insert(
21630                "PROMPT_FILE".into(),
21631                captured.to_string_lossy().into_owned(),
21632            );
21633            let m = posted(&mut f).await;
21634            assert!(m.body.contains("Adds `own.txt`"));
21635            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21636            assert!(m.body.contains("R1-1-1"));
21637            assert!(
21638                m.body
21639                    .contains("Gate (`verify.gate`): passed on the final commit")
21640            );
21641            assert!(
21642                m.body
21643                    .contains("Tests (`verify.e2e`): passed on the final commit")
21644            );
21645            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21646            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21647            let prompt = std::fs::read_to_string(captured).unwrap();
21648            assert!(prompt.contains(&format!(
21649                "git diff {} {head}",
21650                f.runner.state.change_summary.as_ref().unwrap().base
21651            )));
21652            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21653            assert_eq!(
21654                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21655                "二回目の内部修正報告です。"
21656            );
21657        }
21658    }
21659
21660    #[tokio::test]
21661    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21662        for review_only in [false, true] {
21663            let mut f = summary_fixture();
21664            if !review_only {
21665                f.runner.state.candidates[0].agent = "alpha".into();
21666            }
21667            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21668            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21669            std::fs::write(&f.reply, "no usable description").unwrap();
21670            let failed = posted(&mut f).await;
21671            assert!(
21672                failed
21673                    .body
21674                    .contains("No description of what this change does could be generated")
21675            );
21676            assert!(failed.body.is_ascii());
21677            assert!(f.runner.state.github_text.is_none());
21678            assert!(f.runner.state.change_summary.is_none());
21679            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21680            let evidence = crate::pr_outcome::summary_evidence(
21681                &f.runner.state,
21682                crate::pr_outcome::FinalHead::At(&head),
21683            );
21684            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21685            assert!(
21686                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21687                    .exists()
21688            );
21689            f.runner.state =
21690                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21691            std::fs::write(
21692                &f.reply,
21693                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21694            )
21695            .unwrap();
21696            let retried = posted(&mut f).await;
21697            assert!(retried.body.contains("Adds the committed own file."));
21698            assert_eq!(
21699                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21700                "日本語の修正報告"
21701            );
21702        }
21703    }
21704
21705    #[tokio::test]
21706    async fn same_head_verification_and_findings_invalidate_public_prose() {
21707        let mut f = summary_fixture();
21708        posted(&mut f).await;
21709        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21710        let original = f
21711            .runner
21712            .state
21713            .change_summary
21714            .as_ref()
21715            .unwrap()
21716            .evidence
21717            .clone();
21718        ok_gate(&mut f.runner.state, Some(&head));
21719        std::fs::write(
21720            &f.reply,
21721            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21722        )
21723        .unwrap();
21724        let verified = posted(&mut f).await;
21725        assert!(verified.body.contains("now verified"));
21726        assert_ne!(
21727            f.runner.state.change_summary.as_ref().unwrap().evidence,
21728            original
21729        );
21730        let mut round = round_with_notes(1, Some("内部報告"));
21731        round.reviews.push(ReviewRecord {
21732            attempts: 0,
21733            reviewer: 1,
21734            agent: "alpha".into(),
21735            summary: "内部レビュー".into(),
21736            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21737            vote: None,
21738            failed: None,
21739            duration_ms: 0,
21740        });
21741        round
21742            .fix
21743            .as_mut()
21744            .unwrap()
21745            .rejected
21746            .push(crate::verdict::Rejection {
21747                id: "R1-1-1".into(),
21748                why: "呼び出し元から到達できない".into(),
21749            });
21750        f.runner.state.reviews = vec![round];
21751        // A description omitting the required ID is unusable, even with a title.
21752        let omitted = posted(&mut f).await;
21753        assert!(omitted.body.contains("No description"));
21754        assert!(omitted.body.contains("R1-1-1"));
21755        assert!(!omitted.body.contains("到達"));
21756        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21757        let described = posted(&mut f).await;
21758        assert!(described.body.contains("unreachable from callers"));
21759        assert!(described.body.contains("R1-1-1"));
21760        assert!(!described.body.contains("到達"));
21761        let evidence = f
21762            .runner
21763            .state
21764            .change_summary
21765            .as_ref()
21766            .unwrap()
21767            .evidence
21768            .clone();
21769        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21770        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21771        assert!(posted(&mut f).await.body.contains("rationale updated"));
21772        assert_ne!(
21773            f.runner.state.change_summary.as_ref().unwrap().evidence,
21774            evidence
21775        );
21776        // Old serialized summaries have no evidence fingerprint and must regenerate.
21777        f.runner
21778            .state
21779            .change_summary
21780            .as_mut()
21781            .unwrap()
21782            .evidence
21783            .clear();
21784        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21785        assert!(posted(&mut f).await.body.contains("current specification"));
21786    }
21787
21788    #[tokio::test]
21789    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21790        let mut f = summary_fixture();
21791        let m = posted(&mut f).await;
21792        assert_eq!(m.title, "feat(own): add the own file");
21793        assert!(
21794            m.body
21795                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21796            "{}",
21797            m.body
21798        );
21799        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21800        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21801        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21802        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21803        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21804        let kept = f.runner.state.change_summary.as_ref().unwrap();
21805        assert_eq!(
21806            (kept.head.as_str(), kept.base.as_str()),
21807            (head.as_str(), base.as_str())
21808        );
21809        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21810    }
21811
21812    #[tokio::test]
21813    async fn review_only_summary_receives_the_configured_github_language() {
21814        let mut f = summary_fixture();
21815        f.runner.state.config.graph.github_language = "ja".into();
21816        let captured = f.repo.join("summary-prompt.txt");
21817        let author = &mut f.runner.state.config.agents[0];
21818        author.command = vec![
21819            "sh".into(),
21820            "-c".into(),
21821            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21822        ];
21823        author.env.insert(
21824            "PROMPT_FILE".into(),
21825            captured.to_string_lossy().into_owned(),
21826        );
21827        std::fs::write(
21828            &f.reply,
21829            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21830        )
21831        .unwrap();
21832        let m = posted(&mut f).await;
21833        let prompt = std::fs::read_to_string(captured).unwrap();
21834        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21835        assert!(!prompt.contains("in English"));
21836        assert_eq!(m.title, "feat(own): ファイルを追加");
21837        assert!(m.body.contains("own.txt を追加します。"));
21838    }
21839
21840    #[tokio::test]
21841    async fn review_fixes_come_after_the_whole_change_summary() {
21842        let mut f = summary_fixture();
21843        f.runner.state.reviews = vec![
21844            round_with_notes(1, Some("handled the PATH case")),
21845            round_with_notes(2, None),
21846        ];
21847        let m = posted(&mut f).await;
21848        let summary = m.body.find("## Summary").expect("summary");
21849        let fixes = m.body.find("## Review rounds").expect("review rounds");
21850        assert!(summary < fixes, "{}", m.body);
21851        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21852        assert!(m.body.contains("## Final verification"), "{}", m.body);
21853    }
21854
21855    #[tokio::test]
21856    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21857        let mut f = summary_fixture();
21858        let first = posted(&mut f).await;
21859        let first_head = f.runner.state.change_summary.clone().unwrap().head;
21860        // The same head reuses the record: a different reply is not read.
21861        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21862        let again = posted(&mut f).await;
21863        assert_eq!((again.title, again.body), (first.title, first.body));
21864
21865        let g = |args: &[&str]| {
21866            let out = std::process::Command::new("git")
21867                .args(args)
21868                .current_dir(&f.repo)
21869                .quiet()
21870                .output()
21871                .expect("spawn git");
21872            assert!(out.status.success(), "git {args:?}");
21873        };
21874        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21875        g(&["add", "-A"]);
21876        g(&["commit", "-q", "-m", "fix(own): handle more"]);
21877        let second = posted(&mut f).await;
21878        let kept = f.runner.state.change_summary.clone().unwrap();
21879        assert_ne!(kept.head, first_head);
21880        assert!(second.body.contains("Not read yet."), "{}", second.body);
21881        assert!(!second.body.contains("so the first summary is read"));
21882        assert_eq!(second.title, "feat(own): other");
21883    }
21884
21885    #[tokio::test]
21886    async fn an_unavailable_summary_is_stated_and_never_invented() {
21887        let mut f = summary_fixture();
21888        std::fs::write(&f.reply, "no title line here").unwrap();
21889        let m = posted(&mut f).await;
21890        assert!(
21891            m.body
21892                .contains("No description of what this change does could be generated"),
21893            "{}",
21894            m.body
21895        );
21896        assert!(m.body.contains("internal reports remain in local records"));
21897        assert!(f.runner.state.change_summary.is_none());
21898        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21899        // A preview asks nobody and claims nothing.
21900        let winner = f.runner.state.candidates[0].clone();
21901        let preview = f
21902            .runner
21903            .guarded_pr_message(&winner, None, false)
21904            .await
21905            .unwrap()
21906            .unwrap();
21907        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21908    }
21909
21910    #[test]
21911    fn pr_message_review_single_commit_uses_its_subject() {
21912        let state = review_state(&["feat(nats): per-role user"]);
21913        let m = pr_message(&state, 'A');
21914        assert_eq!(m.title, "feat(nats): per-role user");
21915        assert!(!m.body.contains("Review the work already"), "{}", m.body);
21916        assert!(m.body.contains("## Commits under review"), "{}", m.body);
21917    }
21918
21919    #[test]
21920    fn pr_message_review_multi_commit_takes_the_oldest() {
21921        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21922        let m = pr_message(&state, 'A');
21923        assert_eq!(m.title, "feat: the change");
21924        for s in ["feat: the change", "fix: typo", "fix: again"] {
21925            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21926        }
21927    }
21928
21929    #[test]
21930    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21931        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21932            let mut state = review_state(&[first, "fix: later fixup"]);
21933            state.candidates[0].branch = "機能/ログイン".to_owned();
21934            let m = pr_message(&state, 'A');
21935            assert!(
21936                m.title.starts_with("chore: land candidate A of run"),
21937                "{}",
21938                m.title
21939            );
21940        }
21941    }
21942
21943    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21944        BranchFacts {
21945            commits: commits
21946                .iter()
21947                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21948                .collect(),
21949            stat: stat.to_owned(),
21950        }
21951    }
21952
21953    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
21954        let mut r = review_round(true, 0, 1, 1, true, true);
21955        r.round = round;
21956        r.fix = notes.map(|n| FixRecord {
21957            agent: "fixer".to_owned(),
21958            addressed: Vec::new(),
21959            rejected: Vec::new(),
21960            notes: n.to_owned(),
21961            committed: true,
21962            failed: None,
21963            duration_ms: 0,
21964            continuation: None,
21965        });
21966        r
21967    }
21968
21969    #[test]
21970    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
21971        let state = review_state(&["ignored"]);
21972        let f = facts(
21973            &[
21974                (
21975                    "fix(login): resolve PATH on macOS",
21976                    "Login shells skip rc files.",
21977                ),
21978                ("fix: address review", ""),
21979            ],
21980            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
21981        );
21982        let m = pr_message_with(&state, 'A', Some(&f));
21983        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
21984        assert!(
21985            m.body
21986                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
21987            "{}",
21988            m.body
21989        );
21990        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
21991        assert!(m.body.contains("## Diff stat"), "{}", m.body);
21992        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
21993        for banned in [
21994            "Review the work already",
21995            "no task statement",
21996            "Original task",
21997        ] {
21998            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
21999        }
22000        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22001    }
22002
22003    #[test]
22004    fn pr_message_review_truncates_a_huge_first_commit_body() {
22005        let state = review_state(&["ignored"]);
22006        let f = facts(
22007            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22008            "s",
22009        );
22010        let m = pr_message_with(&state, 'A', Some(&f));
22011        assert!(m.body.len() < 30_000, "{}", m.body.len());
22012        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22013        assert!(
22014            m.body.contains("- ... 1 more commit(s)"),
22015            "{}",
22016            m.body.len()
22017        );
22018        assert!(m.body.ends_with("magi:candidate-a\n"));
22019    }
22020
22021    #[test]
22022    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22023        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22024        assert_eq!(m.title, "feat: x");
22025        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22026        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22027        assert!(!m.body.contains("no task statement"), "{}", m.body);
22028    }
22029
22030    #[test]
22031    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22032        let mut state = review_state(&["日本語の件名"]);
22033        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22034        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22035        state.candidates[0].branch = "機能/ログイン".to_owned();
22036        assert!(
22037            pr_message(&state, 'A')
22038                .title
22039                .starts_with("chore: land candidate A")
22040        );
22041    }
22042
22043    #[test]
22044    fn pr_message_review_fixes_survive_a_clean_final_round() {
22045        let mut state = review_state(&["feat: x"]);
22046        state.reviews = vec![
22047            round_with_notes(1, Some("handled the PATH case")),
22048            round_with_notes(2, None),
22049        ];
22050        let body = pr_message(&state, 'A').body;
22051        assert!(
22052            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22053            "{body}"
22054        );
22055        assert!(body.contains("### Round 1"), "{body}");
22056
22057        state.reviews = vec![
22058            round_with_notes(1, Some("first fix")),
22059            round_with_notes(2, Some("")),
22060            round_with_notes(3, Some("second fix")),
22061            round_with_notes(4, None),
22062        ];
22063        let body = pr_message(&state, 'A').body;
22064        assert!(
22065            body.contains("### Round 1") && !body.contains("first fix"),
22066            "{body}"
22067        );
22068        assert!(
22069            body.contains("### Round 3") && !body.contains("second fix"),
22070            "{body}"
22071        );
22072        assert!(body.contains("### Round 2"), "{body}");
22073    }
22074
22075    const FINAL: &str = "abcdef1234567890";
22076
22077    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22078        let mut r = round_with_notes(round, None);
22079        r.verified_head = verified.map(str::to_owned);
22080        r
22081    }
22082
22083    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22084        state.gate = vec![CommandOutcome {
22085            command: "gate".to_owned(),
22086            code: Some(0),
22087            output_tail: String::new(),
22088            duration_ms: 0,
22089            resource_blocked: false,
22090        }];
22091        state.gate_ran = true;
22092        state.gate_head = head.map(str::to_owned);
22093    }
22094
22095    fn body_at(state: &RunState) -> String {
22096        pr_message_raw(
22097            state,
22098            'A',
22099            None,
22100            crate::pr_outcome::FinalHead::At(FINAL),
22101            ChangeNote::NotAsked,
22102        )
22103        .body
22104    }
22105
22106    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22107
22108    #[test]
22109    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22110        let mut state = review_state(&["feat: x"]);
22111        state.reviews = vec![
22112            round_with_notes(1, Some(SANDBOXED)),
22113            passing_round(2, Some(FINAL)),
22114        ];
22115        ok_gate(&mut state, Some(FINAL));
22116        let body = body_at(&state);
22117        assert!(!body.contains("Handled the PATH case."), "{body}");
22118        assert!(!body.contains("could not commit"), "{body}");
22119        assert!(!body.contains("could not run"), "{body}");
22120        assert!(
22121            body.contains("The fix is committed on the branch"),
22122            "{body}"
22123        );
22124        assert!(!body.contains("sandbox"), "{body}");
22125        assert!(
22126            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22127            "{body}"
22128        );
22129        assert!(
22130            body.contains("Gate (`verify.gate`): passed on the final commit"),
22131            "{body}"
22132        );
22133        // The raw report is untouched in the record.
22134        assert!(
22135            state.reviews[0]
22136                .fix
22137                .as_ref()
22138                .unwrap()
22139                .notes
22140                .contains("could not commit")
22141        );
22142    }
22143
22144    #[test]
22145    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22146        let mut state = review_state(&["feat: x"]);
22147        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22148        r1.fix.as_mut().unwrap().committed = false;
22149        let mut r2 = passing_round(2, Some(FINAL));
22150        r2.e2e[0].code = Some(1);
22151        state.reviews = vec![r1, r2];
22152        let body = body_at(&state);
22153        assert!(!body.contains("could not commit"), "{body}");
22154        assert!(!body.contains("could not run"), "{body}");
22155        assert!(body.contains("This round produced no commit"), "{body}");
22156        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22157        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22158        assert!(!body.contains("passed"), "{body}");
22159    }
22160
22161    #[test]
22162    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22163        let mut state = review_state(&["feat: x"]);
22164        state.reviews = vec![
22165            round_with_notes(1, Some(SANDBOXED)),
22166            passing_round(2, Some("0123456789abcdef")),
22167        ];
22168        ok_gate(&mut state, Some("0123456789abcdef"));
22169        let body = body_at(&state);
22170        assert!(!body.contains("HTTP tests could not run"), "{body}");
22171        // The commit claim is still settled by the structured record.
22172        assert!(!body.contains("could not commit"), "{body}");
22173        assert!(
22174            body.contains("passed on an earlier commit (`0123456`)"),
22175            "{body}"
22176        );
22177        assert!(!body.contains("passed on the final commit"), "{body}");
22178
22179        // A gate recorded before `gate_head` existed is "not recorded".
22180        ok_gate(&mut state, None);
22181        let body = body_at(&state);
22182        assert!(
22183            body.contains("passed on a commit that was not recorded"),
22184            "{body}"
22185        );
22186    }
22187
22188    #[test]
22189    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22190        let state = state_with_summary(
22191            "add retries to the client",
22192            "TITLE: feat: retries\n\nDid it.",
22193        );
22194        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22195        assert_eq!(m.title, "feat: retries");
22196        assert!(
22197            m.body.contains("<summary>Original task</summary>"),
22198            "{}",
22199            m.body
22200        );
22201        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22202        assert!(
22203            m.body
22204                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22205            "{}",
22206            m.body
22207        );
22208    }
22209
22210    #[test]
22211    fn pr_message_review_bounds_a_long_english_subject() {
22212        let long = format!("feat: {}", "word ".repeat(100));
22213        let m = pr_message(&review_state(&[&long]), 'A');
22214        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22215        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22216    }
22217
22218    #[test]
22219    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22220        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22221        let m = pr_message(&state, 'A');
22222        assert_eq!(m.title, "add retries");
22223        assert!(m.body.contains("<summary>Original task</summary>"));
22224        assert!(!m.body.contains("Commits under review"));
22225        assert_eq!(landing_subject_source(&state), state.instruction);
22226    }
22227
22228    #[test]
22229    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22230        let state = review_state(&["feat: the change", "fix: typo"]);
22231        let source = landing_subject_source(&state);
22232        assert_eq!(land::merge_subject("", &source), "feat: the change");
22233        assert_eq!(
22234            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22235            "feat: the change"
22236        );
22237        // An operator's rename still wins.
22238        assert_eq!(
22239            land::merge_subject("feat: renamed by hand", &source),
22240            "feat: renamed by hand"
22241        );
22242        let blank = review_state(&["日本語"]);
22243        assert!(
22244            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22245        );
22246    }
22247
22248    #[test]
22249    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22250        let state = review_state(&["feat: the change"]);
22251        let source = landing_subject_source(&state);
22252        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22253        assert_eq!(
22254            land::merge_subject(landing_title(&state, old), &source),
22255            "feat: the change"
22256        );
22257        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22258        let task = state_with_summary("add retries", "");
22259        assert_eq!(landing_title(&task, old), old);
22260    }
22261
22262    #[test]
22263    fn pr_message_describes_the_change_not_the_task() {
22264        let state = state_with_summary(
22265            "今回やってほしいこと: results projector を直す",
22266            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22267        );
22268        let m = pr_message(&state, 'A');
22269        assert_eq!(m.title, "fix(web): batch the runs list reads");
22270        assert!(
22271            m.body.starts_with("## Summary\n\n- reads run.json once"),
22272            "{}",
22273            m.body
22274        );
22275        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22276        let task_at = m.body.find("今回やってほしいこと").unwrap();
22277        let details_at = m.body.find("<details>").unwrap();
22278        assert!(
22279            details_at < task_at,
22280            "the task lives inside <details>: {}",
22281            m.body
22282        );
22283        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22284        assert!(m.body.contains("magi:candidate-a"));
22285    }
22286
22287    #[test]
22288    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22289        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22290        let m = pr_message(&state, 'A');
22291        assert_eq!(m.title, "add retries");
22292        assert!(
22293            m.body.contains("## Summary\n\n- did some things"),
22294            "{}",
22295            m.body
22296        );
22297
22298        let none = RunState::new(
22299            PathBuf::from("/repo"),
22300            "main".to_owned(),
22301            "abc1234".to_owned(),
22302            "add retries".to_owned(),
22303            Config::default(),
22304        );
22305        let m = pr_message(&none, 'A');
22306        assert_eq!(m.title, "add retries");
22307        assert!(!m.body.contains("## Summary"), "{}", m.body);
22308    }
22309
22310    #[test]
22311    fn pr_message_refuses_the_candidate_commit_subject() {
22312        for bad in [
22313            "TITLE: magi: candidate A (uncommitted work)",
22314            "TITLE: chore: stuff (uncommitted work)",
22315            "TITLE:   ",
22316        ] {
22317            let state = state_with_summary("add retries", bad);
22318            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22319        }
22320    }
22321
22322    #[test]
22323    fn pr_message_bounds_a_very_long_task_and_title() {
22324        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22325        let state = state_with_summary(&long, "- nothing");
22326        let m = pr_message(&state, 'A');
22327        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22328        assert!(!m.title.contains('\n'));
22329
22330        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22331        let m = pr_message(&state, 'A');
22332        assert!(m.title.starts_with("feat: "));
22333        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22334        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22335    }
22336
22337    fn long_title_of(instruction: &str) -> String {
22338        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22339    }
22340
22341    #[test]
22342    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22343        let first = "Make the landing path keep a readable title for long tasks";
22344        let line = format!(
22345            "{first}. {}",
22346            "And then keep going with more words ".repeat(20)
22347        );
22348        let t = long_title_of(&line);
22349        assert_eq!(t, first);
22350        assert!(!t.starts_with("chore: land"));
22351    }
22352
22353    #[test]
22354    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22355        let line = "word ".repeat(200);
22356        let t = long_title_of(&line);
22357        assert!(t.ends_with("word..."), "{t}");
22358        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22359    }
22360
22361    #[test]
22362    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22363        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22364            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22365        }
22366    }
22367
22368    #[test]
22369    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22370        let mut state = state_with_summary("再試行を追加する", "- nothing");
22371        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22372        state.config.graph.github_language = "ja".into();
22373        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22374    }
22375
22376    #[test]
22377    fn pr_message_title_limit_is_exact() {
22378        let at = "a".repeat(PR_TITLE_MAX);
22379        assert_eq!(long_title_of(&at), at);
22380        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22381        assert!(over.ends_with("..."), "{over}");
22382        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22383    }
22384
22385    #[test]
22386    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22387        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22388        let t = long_title_of(&line);
22389        assert!(t.ends_with("..."), "{t}");
22390        assert!(t.is_ascii(), "{t}");
22391    }
22392
22393    #[test]
22394    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22395        let line = format!(
22396            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22397            "plus more filler words ".repeat(20)
22398        );
22399        let t = long_title_of(&line);
22400        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22401    }
22402
22403    #[test]
22404    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22405        let line = format!("\n\n# {}", "title words ".repeat(40));
22406        let state = state_with_summary(&line, "- nothing");
22407        let m = pr_message(&state, 'A');
22408        assert!(m.title.starts_with("title words"), "{}", m.title);
22409        assert_eq!(
22410            land::merge_subject(&m.title, &landing_subject_source(&state)),
22411            m.title
22412        );
22413        // An operator's rename wins untouched.
22414        assert_eq!(
22415            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22416            "feat: renamed by hand"
22417        );
22418    }
22419
22420    #[tokio::test]
22421    async fn github_text_rewrite_is_bounded_and_falls_back() {
22422        crate::run::pin_test_home();
22423        for (reply, accepted) in [
22424            (
22425                "TITLE: fix: retries\nAdd retries for failed requests.",
22426                true,
22427            ),
22428            (
22429                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22430                false,
22431            ),
22432            (
22433                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22434                true,
22435            ),
22436            (
22437                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22438                true,
22439            ),
22440            (
22441                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22442                true,
22443            ),
22444            (
22445                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22446                false,
22447            ),
22448            ("TITLE: fix: retries\n", false),
22449        ] {
22450            let dir = tempfile::tempdir().unwrap();
22451            let mut runner = runner_at(RunStatus::Gating);
22452            runner.state = state_with_summary(
22453                "add retries",
22454                "TITLE: fix: retries\n日本語の説明を書きます。",
22455            );
22456            runner.state.repo = dir.path().to_owned();
22457            runner.state.candidates[0].worktree = dir.path().to_owned();
22458            let mut author = spec("alpha");
22459            author.command = vec![
22460                "sh".into(),
22461                "-c".into(),
22462                "printf '%s' \"$REWRITE_REPLY\"".into(),
22463            ];
22464            author.env.insert("REWRITE_REPLY".into(), reply.into());
22465            runner.state.config.agents = vec![author];
22466            seed_public_summary(&mut runner);
22467            let winner = runner.state.candidates[0].clone();
22468            let first = runner
22469                .guarded_pr_message(&winner, None, true)
22470                .await
22471                .unwrap();
22472            let message = if accepted {
22473                first.expect("a passing rewrite never asks")
22474            } else {
22475                // Withheld: the run parks on a question and posts nothing yet.
22476                assert!(first.is_none());
22477                assert!(runner.state.parked);
22478                let store = ask::Questions::open();
22479                let asked: Vec<_> = store
22480                    .list()
22481                    .into_iter()
22482                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22483                    .collect();
22484                assert_eq!(asked.len(), 1);
22485                assert!(!asked[0].detail.contains("secret"));
22486                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22487                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22488                assert_eq!(
22489                    asked[0]
22490                        .choices
22491                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22492                    rewrite.is_some()
22493                );
22494                // Silence: the question lapses and the neutral text is used,
22495                // without a second rewrite call or a second question.
22496                store
22497                    .update(&asked[0].id, |q| {
22498                        q.abandon("test");
22499                        Ok(())
22500                    })
22501                    .unwrap();
22502                let again = runner
22503                    .guarded_pr_message(&winner, None, true)
22504                    .await
22505                    .unwrap()
22506                    .expect("an abandoned question falls back");
22507                let n = store
22508                    .list()
22509                    .into_iter()
22510                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22511                    .count();
22512                assert_eq!(n, 1, "asked once per rejected text");
22513                again
22514            };
22515            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22516            assert!(!message.body.contains("secret"));
22517            assert!(!message.body.contains("prefix"));
22518            assert!(!message.title.contains("secret"));
22519            assert_eq!(
22520                message.body.contains("Add retries for failed requests."),
22521                accepted
22522            );
22523            assert_eq!(
22524                runner.state.seats["github-text-rewrite"].turns, 1,
22525                "only one rewrite invocation"
22526            );
22527            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22528            if accepted {
22529                assert!(runner.state.github_text.is_none());
22530                assert!(!runner.state.parked);
22531                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22532                assert!(message.body.contains("## Final verification"));
22533                assert!(message.body.contains("Evidence: final diff"));
22534                assert!(
22535                    message
22536                        .body
22537                        .contains(&format!("magi:run/{}", runner.state.id))
22538                );
22539            }
22540            if !accepted {
22541                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22542                assert!(
22543                    message
22544                        .body
22545                        .contains(&format!("magi:run/{}", runner.state.id))
22546                );
22547            }
22548        }
22549    }
22550
22551    fn scripted(id: &str, reply: &str) -> AgentSpec {
22552        let mut a = spec(id);
22553        a.command = vec![
22554            "sh".into(),
22555            "-c".into(),
22556            "printf '%s' \"$REWRITE_REPLY\"".into(),
22557        ];
22558        a.env.insert("REWRITE_REPLY".into(), reply.into());
22559        a
22560    }
22561
22562    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22563    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22564
22565    /// Gate tests inject generated public output, not internal reports.
22566    fn seed_public_summary(runner: &mut Runner) {
22567        let repo = &runner.state.repo;
22568        let g = |args: &[&str]| {
22569            let out = std::process::Command::new("git")
22570                .args(args)
22571                .current_dir(repo)
22572                .quiet()
22573                .output()
22574                .unwrap();
22575            assert!(out.status.success(), "git {args:?}");
22576            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22577        };
22578        g(&["init", "-q", "-b", "main"]);
22579        g(&[
22580            "-c",
22581            "user.name=Test",
22582            "-c",
22583            "user.email=test@example.invalid",
22584            "commit",
22585            "-q",
22586            "--allow-empty",
22587            "-m",
22588            "feat: fixture",
22589        ]);
22590        let head = g(&["rev-parse", "HEAD"]);
22591        g(&["update-ref", "refs/remotes/origin/main", &head]);
22592        let evidence = crate::pr_outcome::summary_evidence(
22593            &runner.state,
22594            crate::pr_outcome::FinalHead::At(&head),
22595        );
22596        let generated = &runner.state.candidates[0].summary;
22597        runner.state.change_summary = Some(crate::run::ChangeSummary {
22598            base: head.clone(),
22599            head,
22600            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22601            title: summary_title(generated).unwrap(),
22602            body: summary_without_title(generated),
22603        });
22604    }
22605
22606    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22607        let mut runner = runner_at(RunStatus::Gating);
22608        runner.state = state_with_summary("add retries", JA_SUMMARY);
22609        if review_only {
22610            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22611        }
22612        runner.state.repo = dir.to_owned();
22613        runner.state.candidates[0].worktree = dir.to_owned();
22614        runner.state.config.agents = agents;
22615        seed_public_summary(&mut runner);
22616        runner
22617    }
22618
22619    fn gate_questions(runner: &Runner) -> usize {
22620        ask::Questions::open()
22621            .list()
22622            .into_iter()
22623            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22624            .count()
22625    }
22626
22627    #[tokio::test]
22628    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22629        crate::run::pin_test_home();
22630        let dir = tempfile::tempdir().unwrap();
22631        let mut runner = rewrite_runner(
22632            dir.path(),
22633            true,
22634            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22635        );
22636        runner.state.config.roles.language_judge =
22637            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22638        let winner = runner.state.candidates[0].clone();
22639        let msg = runner
22640            .guarded_pr_message(&winner, None, true)
22641            .await
22642            .unwrap()
22643            .expect("an English rewrite posts without asking");
22644        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22645        assert!(msg.body.contains("<details>kept</details>"));
22646        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22647        assert_eq!(gate_questions(&runner), 0);
22648        assert!(runner.state.github_text.is_none());
22649        assert!(!runner.state.seats.contains_key("impl-A"));
22650        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22651    }
22652
22653    #[tokio::test]
22654    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22655        crate::run::pin_test_home();
22656        let dir = tempfile::tempdir().unwrap();
22657        let mut runner = rewrite_runner(
22658            dir.path(),
22659            false,
22660            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22661        );
22662        runner.state.config.roles.language_judge =
22663            Some(crate::config::AgentChoice::One("beta".into()));
22664        runner.state.seats.insert(
22665            "impl-A".into(),
22666            SeatState::new("impl-A", "alpha", runner.state.seed),
22667        );
22668        let winner = runner.state.candidates[0].clone();
22669        let msg = runner
22670            .guarded_pr_message(&winner, None, true)
22671            .await
22672            .unwrap()
22673            .expect("rewritten");
22674        assert!(msg.body.contains("<details>kept</details>"));
22675        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22676        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22677        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22678    }
22679
22680    #[tokio::test]
22681    async fn rewrite_reuses_a_resumable_implementer_session() {
22682        crate::run::pin_test_home();
22683        let dir = tempfile::tempdir().unwrap();
22684        let mut runner = rewrite_runner(
22685            dir.path(),
22686            false,
22687            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22688        );
22689        runner.state.config.roles.language_judge =
22690            Some(crate::config::AgentChoice::One("beta".into()));
22691        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22692        seat.turns = 1;
22693        runner.state.seats.insert("impl-A".into(), seat);
22694        let winner = runner.state.candidates[0].clone();
22695        runner
22696            .guarded_pr_message(&winner, None, true)
22697            .await
22698            .unwrap()
22699            .expect("rewritten");
22700        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22701        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22702    }
22703
22704    #[tokio::test]
22705    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22706        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22707        // session, was out of quota, and nothing else was asked.
22708        crate::run::pin_test_home();
22709        let dir = tempfile::tempdir().unwrap();
22710        let mut exhausted = spec("alpha");
22711        exhausted.command = vec!["false".into()];
22712        let mut runner = rewrite_runner(
22713            dir.path(),
22714            false,
22715            vec![exhausted, scripted("beta", EN_REWRITE)],
22716        );
22717        // The implementer's own agent is in the role too: it is not asked twice.
22718        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22719            "alpha".into(),
22720            "beta".into(),
22721        ]));
22722        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22723        seat.turns = 1;
22724        runner.state.seats.insert("impl-A".into(), seat);
22725        let winner = runner.state.candidates[0].clone();
22726        let msg = runner
22727            .guarded_pr_message(&winner, None, true)
22728            .await
22729            .unwrap()
22730            .expect("the role rewrites after the implementer fails");
22731        assert!(msg.body.contains("`src/net.rs`"));
22732        assert_eq!(
22733            runner.state.seats["impl-A"].turns, 2,
22734            "the implementer was tried once"
22735        );
22736        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22737        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22738    }
22739
22740    #[tokio::test]
22741    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22742        crate::run::pin_test_home();
22743        let dir = tempfile::tempdir().unwrap();
22744        let mut broken = spec("bad");
22745        broken.command = vec!["false".into()];
22746        let mut runner = rewrite_runner(
22747            dir.path(),
22748            true,
22749            vec![
22750                broken,
22751                scripted("noformat", "Add retries."),
22752                scripted("good", EN_REWRITE),
22753            ],
22754        );
22755        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22756            "bad".into(),
22757            "noformat".into(),
22758            "good".into(),
22759        ]));
22760        let winner = runner.state.candidates[0].clone();
22761        let msg = runner
22762            .guarded_pr_message(&winner, None, true)
22763            .await
22764            .unwrap()
22765            .expect("the third agent rewrites");
22766        assert!(msg.body.contains("`src/net.rs`"));
22767        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22768        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22769    }
22770
22771    #[tokio::test]
22772    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22773        crate::run::pin_test_home();
22774        // (agents, role): still Japanese; no TITLE; nothing resolvable.
22775        for (agents, role, saved) in [
22776            (
22777                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22778                Some("beta"),
22779                true,
22780            ),
22781            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22782            (Vec::new(), None, false),
22783        ] {
22784            let dir = tempfile::tempdir().unwrap();
22785            let mut runner = rewrite_runner(dir.path(), true, agents);
22786            runner.state.config.roles.language_judge =
22787                role.map(|r| crate::config::AgentChoice::One(r.into()));
22788            let winner = runner.state.candidates[0].clone();
22789            let first = runner
22790                .guarded_pr_message(&winner, None, true)
22791                .await
22792                .unwrap();
22793            assert!(first.is_none(), "withheld text parks the run");
22794            assert!(runner.state.parked);
22795            assert_eq!(gate_questions(&runner), 1);
22796            let g = runner.state.github_text.as_ref().unwrap();
22797            assert_eq!(g.rewrite.is_some(), saved);
22798            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22799        }
22800    }
22801
22802    #[tokio::test]
22803    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22804        crate::run::pin_test_home();
22805        let dir = tempfile::tempdir().unwrap();
22806        let mut runner = runner_at(RunStatus::Gating);
22807        runner.state =
22808            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22809        runner.state.repo = dir.path().to_owned();
22810        runner.state.candidates[0].worktree = dir.path().to_owned();
22811        let mut author = spec("alpha");
22812        author.command = vec![
22813            "sh".into(),
22814            "-c".into(),
22815            "printf '%s' \"$REWRITE_REPLY\"".into(),
22816        ];
22817        author.env.insert(
22818            "REWRITE_REPLY".into(),
22819            format!(
22820                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22821                "English sentence. ".repeat(600)
22822            ),
22823        );
22824        runner.state.config.agents = vec![author];
22825        seed_public_summary(&mut runner);
22826        let winner = runner.state.candidates[0].clone();
22827        assert!(
22828            runner
22829                .guarded_pr_message(&winner, None, true)
22830                .await
22831                .unwrap()
22832                .is_none()
22833        );
22834        let record = runner.state.github_text.clone().unwrap();
22835        let rewrite = record.rewrite.clone().unwrap();
22836        assert_eq!(rewrite.failed, ["body-language"]);
22837        assert!(
22838            rewrite
22839                .body
22840                .ends_with(&format!("magi:run/{}", runner.state.id))
22841        );
22842        let store = ask::Questions::open();
22843        let q = store.get(record.question.as_ref().unwrap()).unwrap();
22844        assert!(
22845            q.choices
22846                .contains(&crate::github_text::USE_REWRITE.to_owned())
22847        );
22848        for label in [
22849            "ORIGINAL",
22850            "FALLBACK",
22851            "REWRITE",
22852            "Rules still failed: body-language",
22853            "truncated",
22854        ] {
22855            assert!(q.detail.contains(label), "{label}");
22856        }
22857        let artifact = std::fs::read_to_string(crate::run::artifact_path(
22858            &runner.state,
22859            &crate::github_text::artifact_name(&record.fingerprint),
22860        ))
22861        .unwrap();
22862        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22863        // Restart from serialized state, then retry after a bad owner title.
22864        runner.state =
22865            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22866        store
22867            .update(&q.id, |q| {
22868                q.say("まだ日本語")?;
22869                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22870            })
22871            .unwrap();
22872        assert!(
22873            runner
22874                .guarded_pr_message(&winner, None, true)
22875                .await
22876                .unwrap()
22877                .is_none()
22878        );
22879        let retry = runner.state.github_text.as_ref().unwrap();
22880        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22881        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22882        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22883        assert!(
22884            q.posting_comparison
22885                .as_ref()
22886                .unwrap()
22887                .matches_detail(&q.detail)
22888        );
22889        store
22890            .update(&q.id, |q| {
22891                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22892            })
22893            .unwrap();
22894        let message = runner
22895            .guarded_pr_message(&winner, None, true)
22896            .await
22897            .unwrap()
22898            .unwrap();
22899        assert_eq!(message.title, rewrite.title);
22900        assert_eq!(message.body, rewrite.body);
22901        assert_eq!(
22902            runner.state.github_text.as_ref().unwrap().choice,
22903            Some(GithubTextChoice::Rewrite)
22904        );
22905        let again = runner
22906            .guarded_pr_message(&winner, None, true)
22907            .await
22908            .unwrap()
22909            .unwrap();
22910        assert_eq!(again.title, message.title);
22911        assert_eq!(again.body, message.body);
22912        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22913        assert!(
22914            !runner
22915                .state
22916                .events
22917                .iter()
22918                .any(|e| e.message.contains(&rewrite.title)
22919                    || e.message.contains("English sentence."))
22920        );
22921    }
22922
22923    #[tokio::test]
22924    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22925        crate::run::pin_test_home();
22926        let dir = tempfile::tempdir().unwrap();
22927        let mut runner = runner_at(RunStatus::Gating);
22928        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22929        runner.state.repo = dir.path().to_owned();
22930        runner.state.candidates[0].worktree = dir.path().to_owned();
22931        seed_public_summary(&mut runner);
22932        let winner = runner.state.candidates[0].clone();
22933        assert!(
22934            runner
22935                .guarded_pr_message(&winner, None, true)
22936                .await
22937                .unwrap()
22938                .is_none()
22939        );
22940        let record = runner.state.github_text.as_ref().unwrap();
22941        assert!(record.rewrite.is_none());
22942        // Records written before the additive field deserialize with no rewrite.
22943        let mut old = serde_json::to_value(record).unwrap();
22944        old.as_object_mut().unwrap().remove("rewrite");
22945        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22946        assert!(old.rewrite.is_none());
22947        let id = record.question.clone().unwrap();
22948        ask::Questions::open()
22949            .update(&id, |q| {
22950                q.choices.push(crate::github_text::USE_REWRITE.into());
22951                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22952            })
22953            .unwrap();
22954        let message = runner
22955            .guarded_pr_message(&winner, None, true)
22956            .await
22957            .unwrap()
22958            .unwrap();
22959        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22960        assert_eq!(
22961            runner.state.github_text.as_ref().unwrap().choice,
22962            Some(GithubTextChoice::Fallback)
22963        );
22964    }
22965
22966    #[tokio::test]
22967    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
22968        crate::run::pin_test_home();
22969        let dir = tempfile::tempdir().unwrap();
22970        let mut runner = runner_at(RunStatus::Gating);
22971        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
22972        runner.state.repo = dir.path().to_owned();
22973        runner.state.candidates[0].worktree = dir.path().to_owned();
22974        seed_public_summary(&mut runner);
22975        let winner = runner.state.candidates[0].clone();
22976        assert!(
22977            runner
22978                .guarded_pr_message(&winner, None, true)
22979                .await
22980                .unwrap()
22981                .is_none()
22982        );
22983        let store = ask::Questions::open();
22984        let find = |run: &str| {
22985            store
22986                .list()
22987                .into_iter()
22988                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
22989                .unwrap()
22990        };
22991        let run = runner.state.id.clone();
22992        let q = find(&run);
22993        let fp = runner
22994            .state
22995            .github_text
22996            .as_ref()
22997            .unwrap()
22998            .fingerprint
22999            .clone();
23000        let name = crate::github_text::artifact_name(&fp);
23001        let path = crate::run::artifact_path(&runner.state, &name);
23002        assert!(q.detail.contains(&path.display().to_string()));
23003        assert!(q.detail.contains("title-language: title"));
23004        let saved = std::fs::read_to_string(&path).expect("artifact written");
23005        assert!(saved.contains("日本語のタイトル"));
23006        assert!(q.cwd.is_none());
23007        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23008        // A bad replacement is refused and asked about once more.
23009        let say = |id: &str, text: &str, choice: &str| {
23010            store
23011                .update(id, |q| {
23012                    q.thread.push(ask::Turn {
23013                        who: ask::Who::Operator,
23014                        body: text.to_owned(),
23015                        at: jiff::Timestamp::now(),
23016                        note: None,
23017                    });
23018                    q.answer(ask::Answer::Choice(choice.to_owned()))
23019                })
23020                .unwrap();
23021        };
23022        say(&q.id, "まだ日本語", "use my text");
23023        assert!(
23024            runner
23025                .guarded_pr_message(&winner, None, true)
23026                .await
23027                .unwrap()
23028                .is_none()
23029        );
23030        let second = find(&run);
23031        assert_ne!(second.id, q.id);
23032        assert!(second.detail.contains("did not pass"));
23033        // The retry still compares the original text; the rejected
23034        // replacement is named by category only.
23035        assert!(second.detail.contains("日本語のタイトル"));
23036        assert!(!second.detail.contains("まだ日本語"));
23037        assert!(second.detail.contains(&path.display().to_string()));
23038        say(&second.id, "fix: retry failed requests", "use my text");
23039        let message = runner
23040            .guarded_pr_message(&winner, None, true)
23041            .await
23042            .unwrap()
23043            .expect("a vetted replacement is posted");
23044        assert_eq!(message.title, "fix: retry failed requests");
23045        assert_eq!(
23046            runner
23047                .state
23048                .github_text
23049                .as_ref()
23050                .unwrap()
23051                .chosen_title
23052                .as_deref(),
23053            Some("fix: retry failed requests")
23054        );
23055        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23056        // Resumed again: the saved decision is applied, nothing is re-asked.
23057        let again = runner
23058            .guarded_pr_message(&winner, None, true)
23059            .await
23060            .unwrap()
23061            .unwrap();
23062        assert_eq!(again.title, message.title);
23063        assert_eq!(
23064            store
23065                .list()
23066                .into_iter()
23067                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23068                .count(),
23069            2
23070        );
23071    }
23072
23073    #[test]
23074    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23075        // What magi itself writes stays English under any configured language,
23076        // so a future localisation of these headings fails here. (The agents'
23077        // own text is also checked by the posting gate.)
23078        let mut state = state_with_summary(
23079            "add retries",
23080            "TITLE: fix(web): batch reads\n- reads run.json once",
23081        );
23082        state.config.graph.language = "ja".to_owned();
23083        let m = pr_message(&state, 'A');
23084        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23085        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23086
23087        // The task is the operator's own text: it goes in untouched, and the
23088        // fallback title (no summary) may be in its language too.
23089        let task = "今回やってほしいこと: results projector を直す";
23090        let mut state = state_with_summary(task, "- no title line");
23091        state.config.graph.language = "ja".to_owned();
23092        let m = pr_message(&state, 'A');
23093        assert_eq!(
23094            m.title,
23095            format!("chore: land candidate A of run {}", state.id)
23096        );
23097        assert!(
23098            m.body.contains(&format!(
23099                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23100            )),
23101            "{}",
23102            m.body
23103        );
23104    }
23105
23106    #[test]
23107    fn pr_message_scrubs_home_paths_and_addresses() {
23108        let state = state_with_summary(
23109            "fix it in /Users/someone/src/x",
23110            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23111        );
23112        let m = pr_message(&state, 'A');
23113        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23114            assert!(!m.body.contains(leak), "{}", m.body);
23115        }
23116        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23117    }
23118
23119    #[test]
23120    fn pr_message_survives_a_task_that_closes_details() {
23121        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23122        let m = pr_message(&state, 'A');
23123        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23124    }
23125
23126    #[test]
23127    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23128        let cmd = manual_merge_command(
23129            MergeStyle::Squash,
23130            Path::new("/repo"),
23131            "b",
23132            "fix: \"quoted\" $(x) `y`\n\nbody",
23133        );
23134        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23135    }
23136
23137    #[test]
23138    fn manual_merge_command_matches_the_configured_style() {
23139        let repo = Path::new("/repo");
23140        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23141
23142        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23143        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23144
23145        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23146        assert_eq!(
23147            squash,
23148            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23149             \"Merge magi run 0832 (candidate A)\""
23150        );
23151
23152        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23153        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23154    }
23155
23156    #[test]
23157    fn a_nudge_gets_a_quarter_of_the_budget() {
23158        // The judge and implement budgets magi ships with.
23159        assert_eq!(retry_budget(secs(1200), true), secs(300));
23160        assert_eq!(retry_budget(secs(3600), true), secs(900));
23161    }
23162
23163    #[test]
23164    fn a_resent_prompt_keeps_the_whole_budget() {
23165        // The seat kept no context, so the retry is the original job again and
23166        // shortening it would only guarantee a second failure.
23167        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23168        assert_eq!(retry_budget(secs(60), false), secs(60));
23169    }
23170
23171    #[test]
23172    fn the_floor_never_exceeds_the_original_budget() {
23173        // A short configured timeout must not be *raised* by the floor: the
23174        // operator asked for a bound, and a retry may not outlast the attempt
23175        // it is retrying.
23176        assert_eq!(retry_budget(secs(60), true), secs(60));
23177        assert_eq!(retry_budget(secs(480), true), secs(120));
23178        assert_eq!(retry_budget(secs(0), true), secs(0));
23179    }
23180
23181    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23182        agent::CommandEvidence {
23183            id: "item1".to_owned(),
23184            description: "cargo test".to_owned(),
23185            exit_code,
23186            result_summary: String::new(),
23187            source: "codex".to_owned(),
23188        }
23189    }
23190
23191    #[test]
23192    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23193        // No evidence is not the same fact as unconfirmed evidence: a
23194        // backend with no adapter, or a reply that ran no commands at all,
23195        // must not be misread as carrying a dangling job.
23196        assert!(!has_unconfirmed_command(&[]));
23197    }
23198
23199    #[test]
23200    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23201        // Deliberately not a check on the exit code's *value*: a fixer
23202        // legitimately runs something that fails mid-iteration before it
23203        // succeeds, and that must never by itself reopen a valid report.
23204        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23205        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23206        assert!(!has_unconfirmed_command(&[
23207            evidence(Some(0)),
23208            evidence(Some(101))
23209        ]));
23210    }
23211
23212    #[test]
23213    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23214        assert!(has_unconfirmed_command(&[
23215            evidence(Some(0)),
23216            evidence(None)
23217        ]));
23218    }
23219
23220    #[test]
23221    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23222        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23223        assert_eq!(
23224            verified_noop_claim(true, &[], text).as_deref(),
23225            Some("already fixed by b32cfc4, on main.")
23226        );
23227    }
23228
23229    #[test]
23230    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23231        // A timeout or a bad exit code reads as the ordinary loss it is,
23232        // whatever the reply's own prose claims.
23233        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23234        assert!(verified_noop_claim(false, &[], text).is_none());
23235    }
23236
23237    #[test]
23238    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23239        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23240        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23241        // A confirmed command alongside the marker is fine.
23242        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23243    }
23244
23245    #[test]
23246    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23247        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23248    }
23249
23250    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23251    /// pair, in order, labelled A, B, C, ...
23252    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23253        runner.state.candidates = shape
23254            .iter()
23255            .enumerate()
23256            .map(|(i, &(empty, verified))| Candidate {
23257                index: i,
23258                label: (b'A' + i as u8) as char,
23259                agent: "sonnet".to_owned(),
23260                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23261                worktree: PathBuf::from(format!("/wt/{i}")),
23262                summary: String::new(),
23263                stat: String::new(),
23264                files: 0,
23265                commits: 0,
23266                empty,
23267                failed: None,
23268                verified_noop: verified.map(str::to_owned),
23269                duration_ms: 0,
23270                folded: false,
23271            })
23272            .collect();
23273    }
23274
23275    #[test]
23276    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23277        ask_test_home();
23278        let mut runner = runner_at(RunStatus::Implementing);
23279        set_candidates(
23280            &mut runner,
23281            &[
23282                (true, Some("already on main at b32cfc4")),
23283                (true, Some("same fix, see the existing test")),
23284            ],
23285        );
23286
23287        runner
23288            .after_implement()
23289            .expect("a verified no-op is not an error");
23290
23291        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23292    }
23293
23294    #[test]
23295    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23296        ask_test_home();
23297        let mut runner = runner_at(RunStatus::Implementing);
23298        // Candidate A declares a verified no-op; candidate B simply wrote
23299        // nothing and said nothing about why. One candidate's claim is not
23300        // the whole run's agreement.
23301        set_candidates(
23302            &mut runner,
23303            &[(true, Some("already on main at b32cfc4")), (true, None)],
23304        );
23305
23306        let err = runner
23307            .after_implement()
23308            .expect_err("an unverified empty candidate must still fail the run");
23309
23310        assert!(
23311            err.to_string().contains("no candidate produced a change"),
23312            "{err}"
23313        );
23314        assert_eq!(runner.state.status, RunStatus::Failed);
23315    }
23316
23317    #[test]
23318    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23319        ask_test_home();
23320        let mut runner = runner_at(RunStatus::Implementing);
23321        set_candidates(&mut runner, &[(true, None), (true, None)]);
23322
23323        let err = runner
23324            .after_implement()
23325            .expect_err("no candidate declared anything; this is an ordinary failure");
23326
23327        assert!(
23328            err.to_string().contains("no candidate produced a change"),
23329            "{err}"
23330        );
23331        assert_eq!(runner.state.status, RunStatus::Failed);
23332    }
23333}
23334
23335/// A graft rollback whose restore could not be confirmed (see
23336/// [`Runner::graft`]).
23337#[derive(Debug)]
23338struct GraftRestoreFailed(String);
23339
23340impl std::fmt::Display for GraftRestoreFailed {
23341    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23342        write!(
23343            f,
23344            "could not restore the winner to its pre-graft commit: {}",
23345            self.0
23346        )
23347    }
23348}
23349
23350impl std::error::Error for GraftRestoreFailed {}
23351
23352/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23353/// not the agent: they never draw on an agent's own timeout, and running out of
23354/// them never reads as an agent failure.
23355#[cfg(test)]
23356mod resource_wait_tests {
23357    use super::*;
23358    use std::collections::BTreeMap;
23359
23360    fn sh(id: &str, script: &str) -> AgentSpec {
23361        AgentSpec {
23362            id: id.to_owned(),
23363            kind: crate::config::AgentKind::Command,
23364            model: None,
23365            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23366            extra_args: Vec::new(),
23367            env: BTreeMap::new(),
23368            prompt_delivery: None,
23369            max_turn_secs: None,
23370        }
23371    }
23372
23373    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23374        SeatJob {
23375            seat: SeatState::new("impl-A", &spec.id, 1),
23376            spec,
23377            cwd: dir.to_path_buf(),
23378            prompt: "do it".to_owned(),
23379            timeout,
23380            allow_write: true,
23381            sessions: false,
23382            artifacts: dir.join("art"),
23383            stem: "impl-A".to_owned(),
23384            handover: None,
23385        }
23386    }
23387
23388    fn state(resource_wait_secs: u64) -> RunState {
23389        let mut config = Config::default();
23390        config.graph.timeout_resource_wait = resource_wait_secs;
23391        RunState::new(
23392            PathBuf::from("/repo"),
23393            "main".to_owned(),
23394            "abc1234".to_owned(),
23395            "task".to_owned(),
23396            config,
23397        )
23398    }
23399
23400    fn hold(cache: &Path) -> crate::cache::Guard {
23401        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23402        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23403            crate::cache::AcquireOutcome::Acquired(g) => g,
23404            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23405        }
23406    }
23407
23408    async fn run(
23409        jobs: Vec<SeatJob>,
23410        sem: Arc<Semaphore>,
23411        cache: Option<&Path>,
23412        st: &mut RunState,
23413    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23414        let prompts = Prompts::default();
23415        let ctx = WaveCtx {
23416            run: "r",
23417            node: "implement",
23418            prompts: &prompts,
23419            cache,
23420            round: None,
23421            carry_seats: false,
23422            pauses: None,
23423        };
23424        wave(jobs, sem, &ctx, st, 0).await
23425    }
23426
23427    #[tokio::test]
23428    async fn verification_cache_wait_preserves_the_command_budget() {
23429        crate::run::pin_test_home();
23430        let tmp = tempfile::tempdir().unwrap();
23431        let cache = tmp.path().join("target");
23432        let guard = hold(&cache);
23433        let release = tokio::spawn(async move {
23434            tokio::time::sleep(Duration::from_millis(100)).await;
23435            guard.release();
23436        });
23437        let mut st = state(10);
23438        let budget = Duration::from_millis(50);
23439        let wait = repository_wait(&st, None);
23440        let (outcomes, _) = with_cache_lease(
23441            &mut st,
23442            Some(&cache),
23443            "gate",
23444            "gate",
23445            tmp.path(),
23446            "abc1234",
23447            budget,
23448            "test gate",
23449            &wait,
23450            |_, available| async move {
23451                assert_eq!(
23452                    available, budget,
23453                    "contention must not shorten verification"
23454                );
23455                (Vec::new(), false, Vec::new())
23456            },
23457        )
23458        .await
23459        .unwrap();
23460        release.await.unwrap();
23461        assert!(outcomes.is_empty(), "{outcomes:?}");
23462        assert!(
23463            st.events
23464                .iter()
23465                .any(|event| event.message.contains("waiting for"))
23466        );
23467        assert!(st.resource_wait.is_none());
23468        // The verification body no longer owns the lease after returning.
23469        hold(&cache).release();
23470    }
23471
23472    #[tokio::test]
23473    async fn verification_cache_wait_expires_without_starting_commands() {
23474        crate::run::pin_test_home();
23475        let tmp = tempfile::tempdir().unwrap();
23476        let cache = tmp.path().join("target");
23477        let _held = hold(&cache);
23478        let mut st = state(0);
23479        let wait = repository_wait(&st, None);
23480        let (outcomes, _) = with_cache_lease(
23481            &mut st,
23482            Some(&cache),
23483            "gate",
23484            "gate",
23485            tmp.path(),
23486            "abc1234",
23487            Duration::from_secs(60),
23488            "test gate",
23489            &wait,
23490            |_, _| async { panic!("commands must not start without the cache lease") },
23491        )
23492        .await
23493        .unwrap();
23494        assert_eq!(outcomes.len(), 1);
23495        assert!(outcomes[0].resource_blocked);
23496        assert!(st.resource_wait.is_none());
23497        assert!(matches!(
23498            st.resource_block.unwrap().cause,
23499            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23500        ));
23501    }
23502
23503    #[tokio::test]
23504    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23505        crate::run::pin_test_home();
23506        let tmp = tempfile::tempdir().unwrap();
23507        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23508        let cache = tmp.path().join("cache");
23509        let mut state = state(21600);
23510        let wait = repository_wait(&state, None);
23511        let (outcomes, _) = with_cache_lease(
23512            &mut state,
23513            Some(&cache),
23514            "gate",
23515            "gate",
23516            tmp.path(),
23517            "head",
23518            Duration::from_secs(60),
23519            "gate",
23520            &wait,
23521            |_, _| async { panic!("freshness failure must not start verification") },
23522        )
23523        .await
23524        .unwrap();
23525        assert_eq!(outcomes.len(), 1);
23526        assert!(outcomes[0].resource_blocked);
23527        let block = state.resource_block.unwrap();
23528        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23529        assert!(!block.description().contains("timed out"));
23530    }
23531
23532    #[tokio::test]
23533    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23534        crate::run::pin_test_home();
23535        let tmp = tempfile::tempdir().unwrap();
23536        let cache = tmp.path().join("cache");
23537        let _held = hold(&cache);
23538        let mut state = state(21600);
23539        let pause = Pause::new();
23540        let interrupt = Pause::new();
23541        let cause = ParkCause::Operator;
23542        pause.park_for(cause.clone());
23543        let prompts = Prompts::default();
23544        let ctx = WaveCtx {
23545            run: "test",
23546            node: "fix",
23547            prompts: &prompts,
23548            cache: Some(&cache),
23549            round: None,
23550            carry_seats: false,
23551            pauses: Some((&pause, &interrupt)),
23552        };
23553        let result = wave(
23554            vec![job(
23555                sh("writer", "exit 99"),
23556                tmp.path(),
23557                Duration::from_secs(1),
23558            )],
23559            Arc::new(Semaphore::new(1)),
23560            &ctx,
23561            &mut state,
23562            0,
23563        )
23564        .await;
23565        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23566        assert_eq!(result[0].1.turns, 0);
23567        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23568        assert_eq!(
23569            state.resource_block.unwrap().cause,
23570            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23571        );
23572        assert!(state.active.is_empty());
23573    }
23574
23575    #[tokio::test]
23576    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23577        crate::run::pin_test_home();
23578        let tmp = tempfile::tempdir().unwrap();
23579        let cache = tmp.path().join("target-a");
23580        let guard = hold(&cache);
23581        // Held for longer than the agent's whole 1s budget.
23582        tokio::spawn(async move {
23583            tokio::time::sleep(Duration::from_millis(1500)).await;
23584            guard.release();
23585        });
23586        let mut st = state(60);
23587        let results = run(
23588            vec![job(
23589                sh("alpha", "echo ok"),
23590                tmp.path(),
23591                Duration::from_secs(1),
23592            )],
23593            Arc::new(Semaphore::new(1)),
23594            Some(&cache),
23595            &mut st,
23596        )
23597        .await;
23598        assert!(
23599            matches!(results[0].2, AgentOutcome::Ok(_)),
23600            "the wait must not be charged to the agent"
23601        );
23602        assert!(st.resource_block.is_none());
23603        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23604        assert!(st.resource_wait.is_none() && st.active.is_empty());
23605    }
23606
23607    #[tokio::test]
23608    #[cfg(unix)]
23609    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23610        use std::os::fd::AsRawFd as _;
23611        crate::run::pin_test_home();
23612        let tmp = tempfile::tempdir().unwrap();
23613        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23614        let common = tmp.path().join(".git");
23615        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23616        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23617        assert_eq!(
23618            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23619            0
23620        );
23621        let mut st = state(1);
23622        st.config.coordination = crate::config::test_coordination();
23623        let mut reader = job(
23624            sh("reader", "echo must-not-run"),
23625            tmp.path(),
23626            Duration::from_secs(30),
23627        );
23628        reader.allow_write = false;
23629        let results = run(
23630            vec![reader.clone()],
23631            Arc::new(Semaphore::new(1)),
23632            None,
23633            &mut st,
23634        )
23635        .await;
23636        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23637        assert_eq!(results[0].1.turns, 0);
23638        assert!(FailClass::of(&results[0].2).is_none());
23639        assert_eq!(
23640            st.resource_block.as_ref().unwrap().kind,
23641            "repository coordination"
23642        );
23643        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23644        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23645        drop(external);
23646        let editor = crate::repo_lock::SeatLease::for_checkout(
23647            tmp.path(),
23648            true,
23649            crate::config::test_coordination().as_ref(),
23650        )
23651        .await
23652        .unwrap();
23653        let pause = Pause::new();
23654        let interrupt = Pause::new();
23655        let prompts = Prompts::default();
23656        let ctx = WaveCtx {
23657            run: "r",
23658            node: "review",
23659            prompts: &prompts,
23660            cache: None,
23661            round: None,
23662            carry_seats: false,
23663            pauses: Some((&pause, &interrupt)),
23664        };
23665        let (results, ()) = tokio::join!(
23666            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23667            async {
23668                tokio::time::sleep(Duration::from_millis(50)).await;
23669                pause.park();
23670            }
23671        );
23672        assert!(
23673            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23674        );
23675        assert_eq!(results[0].1.turns, 0);
23676        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23677        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23678        drop(editor);
23679    }
23680
23681    #[tokio::test]
23682    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23683        crate::run::pin_test_home();
23684        let tmp = tempfile::tempdir().unwrap();
23685        let cache = tmp.path().join("target-b");
23686        let _held = hold(&cache);
23687        let mut st = state(1);
23688        let results = run(
23689            vec![job(
23690                sh("alpha", "echo ok"),
23691                tmp.path(),
23692                Duration::from_secs(1),
23693            )],
23694            Arc::new(Semaphore::new(1)),
23695            Some(&cache),
23696            &mut st,
23697        )
23698        .await;
23699        let (_, seat, out) = &results[0];
23700        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23701        assert!(wave_blocked(&results));
23702        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23703        assert_eq!(seat.turns, 0, "nothing was asked");
23704        assert!(
23705            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23706            "no agent was spawned"
23707        );
23708        assert!(FailClass::of(out).is_none(), "not a failure class");
23709        let block = st.resource_block.as_ref().expect("recorded for the resume");
23710        assert_eq!(block.kind, "shared build cache");
23711        assert!(matches!(
23712            block.cause,
23713            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23714        ));
23715        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23716        assert!(st.active.is_empty());
23717    }
23718
23719    #[tokio::test]
23720    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23721        crate::run::pin_test_home();
23722        let tmp = tempfile::tempdir().unwrap();
23723        let mut st = state(1);
23724        let results = run(
23725            vec![job(
23726                sh("alpha", "echo ok"),
23727                tmp.path(),
23728                Duration::from_secs(1),
23729            )],
23730            Arc::new(Semaphore::new(0)),
23731            None,
23732            &mut st,
23733        )
23734        .await;
23735        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23736        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23737        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23738    }
23739
23740    #[tokio::test]
23741    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23742        crate::run::pin_test_home();
23743        let tmp = tempfile::tempdir().unwrap();
23744        let mut st = state(60);
23745        let results = run(
23746            vec![job(
23747                sh("alpha", "sleep 30"),
23748                tmp.path(),
23749                Duration::from_secs(1),
23750            )],
23751            Arc::new(Semaphore::new(1)),
23752            None,
23753            &mut st,
23754        )
23755        .await;
23756        let out = &results[0].2;
23757        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23758        let class = FailClass::of(out).expect("a failure");
23759        assert_eq!(class, FailClass::Timeout);
23760        assert!(should_hand_over(None, &class), "existing handover policy");
23761        assert!(st.resource_block.is_none());
23762    }
23763}