Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Recorded before the first save, so a crash right after minting
773        // cannot leave a run with no origin. Legibility only: nothing reads it
774        // to decide anything.
775        state.fetched_base = Some(fetched_base);
776        state.origin = Some(origin);
777        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778        state.status = RunStatus::Prep;
779        state.event("start", format!("run {} created", state.id));
780        startup.finish(&mut state, Ok(()))?;
781        Ok(Self {
782            state,
783            roles,
784            sem: Arc::new(Semaphore::new(max_parallel)),
785            pause: Pause::new(),
786            interrupt: Pause::new(),
787        })
788    }
789
790    /// Open a review-only run against work that already exists on `branch`.
791    ///
792    /// The expensive half of the graph is the implement wave — measured at
793    /// 111 and 134 internal tool-loop turns on this repository, against a
794    /// handful for a judge or a reviewer. The cheap half is worth running on
795    /// hand-written work too, and there was no way to reach it.
796    ///
797    /// No new state and no schema change are needed: a run with **one** viable
798    /// candidate and a tally already decided degrades `execute` to exactly
799    /// review → gate → merge, because `judge` skips a single-candidate field,
800    /// `deliberate` has fewer than two first choices to reconcile, `vote`
801    /// returns early, `tally` is already present and `fold_losers` has no
802    /// losers. Resuming such a run therefore does the right thing as well.
803    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804        Self::review_taking_over(repo, branch, config, None, origin).await
805    }
806
807    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
808    /// at the same task still has `branch` checked out, its worktree is
809    /// released first if that is safe (see [`crate::handover`]), and the
810    /// review refuses with the reason if it is not. `None` is a hand-run
811    /// review: it has no earlier attempts, so only a worktree of a dead run
812    /// magi recorded itself can be released.
813    pub async fn review_taking_over(
814        repo: &Path,
815        branch: &str,
816        config: Config,
817        takeover: Option<crate::handover::Takeover>,
818        origin: Origin,
819    ) -> Result<Self> {
820        let repo = git::toplevel(repo).await?;
821        let missing = agent::missing_programs(&config.agents);
822        if !missing.is_empty() {
823            bail!(
824                "these agent programs are not on PATH: {}. Fix the roster in \
825                 magi.toml or install them.",
826                missing.join(", ")
827            );
828        }
829        let base_branch =
830            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831                .await?;
832        if base_branch == branch {
833            bail!("`{branch}` is the base branch; there is nothing to review against");
834        }
835        let fetched_base = resolve_base(
836            &repo,
837            &base_branch,
838            &config.merge.remote,
839            config.coordination.as_ref(),
840        )
841        .await?;
842        let base_commit = fetched_base.commit.clone();
843
844        let roles = config.resolve_roles()?;
845        let max_parallel = config.graph.max_parallel.max(1);
846        let mut state = RunState::new(
847            repo.clone(),
848            base_branch,
849            base_commit.clone(),
850            format!("Review the work already on branch `{branch}`."),
851            config,
852        );
853        state.fetched_base = Some(fetched_base);
854        state.origin = Some(origin);
855        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857        let opened = async {
858            startup.stage(&mut state, "handover")?;
859
860            // Released before anything else touches the branch: a stale local
861            // branch is moved with `git branch -f`, which git refuses while an
862            // earlier attempt's worktree still has it checked out. Everything
863            // after this point that can fail puts the old run back.
864            // A hand-run review has no task, hence no earlier attempts, but a
865            // worktree of a dead run magi made may still be released.
866            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867                earlier: Vec::new(),
868                home: crate::run::home(),
869                choice: None,
870                task_instruction: None,
871            });
872            let wait = repository_wait(&state, None);
873            let coordination_config = state.config.coordination.clone();
874            let coordination =
875                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876                    .await?;
877            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878            if let Some(released) = &released {
879                state.event(
880                    "release",
881                    format!(
882                        "took `{branch}` over from run {}: its worktree was released: {}",
883                        crate::run::short_of(&released.old_id),
884                        released.audit
885                    ),
886                );
887            }
888            let opened = async {
889                startup.stage(&mut state, "review-preflight")?;
890                // The owner's answer to an earlier divergence question is applied
891                // here: after the release (git will not move a checked-out branch)
892                // and before the sync that would otherwise ask again.
893                state.foreign_checkout = crate::handover::preflight_review(
894                    &repo,
895                    branch,
896                    &takeover.home,
897                    &state.config.merge.remote,
898                )
899                .await?;
900                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902                }
903                if let Some(choice) = takeover.choice.as_ref()
904                    && let Err(e) =
905                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906                            .await
907                {
908                    return Err(e.context("applying the owner's answer about the diverged branch"));
909                }
910                // An earlier attempt of this task that published the task head, and
911                // whose work is the branch under review, hands that head over: the
912                // review run updates the same pull request. Without one (a legacy
913                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
914                // own head and its own open pull request is adopted as before.
915                for id in takeover.earlier.iter().rev() {
916                    if let Ok(old) = RunState::load_under(id, &takeover.home)
917                        && let Some(head) = old.pr_head.clone()
918                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919                    {
920                        state.pr_head = Some(head);
921                        break;
922                    }
923                }
924                drop(coordination);
925                startup.stage(&mut state, "open-review")?;
926                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927            }
928            .await;
929            if opened.is_err()
930                && let Some(released) = &released
931            {
932                // Restoration is ownership checked and best effort. Keep the original
933                // startup cause even if reacquiring coordination is no longer possible.
934                match crate::repo_lock::SeatLease::operation_with(
935                    &repo,
936                    coordination_config.as_ref(),
937                    &wait,
938                )
939                .await
940                {
941                    Ok(_coordination) => {
942                        released.restore(&repo, branch).await;
943                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
944                            .map(|old| {
945                                format!(
946                                    "predecessor released_to={:?}; candidate path exists={}",
947                                    old.released_to,
948                                    old.candidates.iter().any(|c| {
949                                        c.branch == branch && c.worktree.exists()
950                                    }),
951                                )
952                            })
953                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955                    }
956                    Err(error) => {
957                        tracing::warn!(
958                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959                            state.id
960                        );
961                        state.event(
962                            "startup-restore",
963                            format!("restoration not attempted: {error:#}"),
964                        );
965                    }
966                }
967            }
968            opened
969        }
970        .await;
971        startup.finish(&mut state, opened)?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Recover exactly the operator-selected public implementation, without touching old sources.
982    pub async fn recover_pr(
983        repo: &Path,
984        task: &str,
985        selection: crate::pr_recovery::Selection,
986        mut config: Config,
987    ) -> Result<Self> {
988        selection.configure(&mut config)?;
989        selection.check_task(task, &selection.instruction)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.origin = Some(crate::run::Origin::queue(task));
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.runs.last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.ensure_base_episode();
1417        state.record_base_sync_stop(&head);
1418        state.save()?;
1419        let fetched =
1420            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421        anyhow::ensure!(
1422            fetched.ok(),
1423            "cannot authorize base recovery without a fetched base"
1424        );
1425        let approved_tip = git::rev_parse(
1426            &state.repo,
1427            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428        )
1429        .await?;
1430        if resumption {
1431            anyhow::ensure!(
1432                crate::daemon::compatible_run_instruction(&state, &queued),
1433                "resumption instruction does not match the retained run"
1434            );
1435            let approval = queued
1436                .resumption
1437                .as_ref()
1438                .context("no operator resumption")?;
1439            anyhow::ensure!(
1440                approval.instruction == queued.instruction,
1441                "resumption approval does not match the current task instruction"
1442            );
1443            state.authorize_base_sync_resumption(task, approval)?;
1444        } else {
1445            state.authorize_base_sync(task)?;
1446        }
1447        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448        state
1449            .base_sync_recovery
1450            .authorizations
1451            .last_mut()
1452            .unwrap()
1453            .scope = Some(crate::base_sync_recovery::Scope {
1454            tip: approved_tip,
1455            instruction: queued.instruction.clone(),
1456            episode,
1457        });
1458        state.event(
1459            "base-recovery",
1460            "owner authorized a bounded base synchronization interval; fresh verification required",
1461        );
1462        state.save()?;
1463        Ok(())
1464    }
1465
1466    /// Reopen an existing run.
1467    pub fn resume(id: &str) -> Result<Self> {
1468        let state = RunState::load(id)?;
1469        if state.startup.as_ref().is_some_and(|s| !s.complete)
1470            || (state.status == RunStatus::Prep
1471                && state.instruction.trim().is_empty()
1472                && state.candidates.is_empty()
1473                && state.prep_complete == Some(false))
1474        {
1475            bail!(
1476                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477            );
1478        }
1479        crate::pr_head::ensure_owner(&state)?;
1480        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481            bail!(
1482                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483            );
1484        }
1485        if let Some(to) = &state.released_to {
1486            bail!(
1487                "run {} cannot be resumed: its worktree was released to run {}",
1488                state.short(),
1489                crate::run::short_of(to)
1490            );
1491        }
1492        let roles = state.config.resolve_roles()?;
1493        if let Some(id) = &state.config.graph.selected_implementer
1494            && state.candidates.iter().any(|c| c.agent != *id)
1495        {
1496            bail!(
1497                "replacement implementer pin does not match the saved candidates; task must stay held"
1498            );
1499        }
1500        let max_parallel = state.config.graph.max_parallel.max(1);
1501        Ok(Self {
1502            state,
1503            roles,
1504            sem: Arc::new(Semaphore::new(max_parallel)),
1505            pause: Pause::new(),
1506            interrupt: Pause::new(),
1507        })
1508    }
1509
1510    /// Walk the graph to a terminal state, skipping nodes already recorded.
1511    ///
1512    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1513    /// the phone - ends here, so this is the one place a run that ended
1514    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1515    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1516    pub async fn execute(&mut self) -> Result<()> {
1517        anyhow::ensure!(
1518            self.state.startup.as_ref().is_none_or(|s| s.complete),
1519            "startup incomplete; inspect retained evidence before explicit recovery"
1520        );
1521        if let Some(selection) = &self.state.pr_recovery
1522            && (selection.source_proof.is_some() || selection.source_run.is_some())
1523        {
1524            Box::pin(selection.verify_source(&self.state.repo)).await?;
1525        }
1526        // Queue claims cover daemon work; this also serializes manual drivers
1527        // consuming a repair answer, including the new review and decision.
1528        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529            || self.state.land_approval.is_some()
1530            || self.state.merge_decision.is_some()
1531            || !self.state.merge_repairs.is_empty()
1532            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533        {
1534            let claim = FixClaim::acquire(&self.state.dir())?;
1535            let disk = RunState::load(&self.state.id)?;
1536            let checkpoint = |s: &RunState| {
1537                (
1538                    s.merge_repairs.len(),
1539                    s.merge_repairs
1540                        .last()
1541                        .map(|r| (r.question.clone(), r.phase)),
1542                    s.land_approval
1543                        .as_ref()
1544                        .map(|a| (a.question.clone(), a.head.clone())),
1545                )
1546            };
1547            if checkpoint(&disk) != checkpoint(&self.state) {
1548                bail!(
1549                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550                );
1551            }
1552            Some(claim)
1553        } else {
1554            None
1555        };
1556        let mut execution = self.claim_driver().await?;
1557        let runner = &mut *execution.runner;
1558        let wait = runner.repository_wait();
1559        // Keep the graph's large future off the callers' stack, including
1560        // standalone task attempts running on a default-sized test thread.
1561        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562        let result = match result {
1563            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564                record_repository_block(&mut runner.state, &e);
1565                runner.park_here()?;
1566                Ok(())
1567            }
1568            other => other,
1569        };
1570        execution.driver.finish(&mut runner.state).await?;
1571        let ended = if result.is_err() {
1572            Some({
1573                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574                if let Err(error) = &result {
1575                    notice.message = format!(
1576                        "Run {} stopped with an error: {}",
1577                        runner.state.id,
1578                        error.to_string().lines().next().unwrap_or("unknown error")
1579                    );
1580                }
1581                notice
1582            })
1583        } else {
1584            crate::notices::run_ended(&runner.state)
1585        };
1586        if let Some(notice) = ended {
1587            crate::notices::raise(notice);
1588        }
1589        result
1590    }
1591
1592    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594    }
1595
1596    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597        let wait = self.repository_wait();
1598        coordinate_operation(&mut self.state, cwd, &wait).await
1599    }
1600
1601    /// Validate ownership and persist the driver claim in one critical section.
1602    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603        let owner = match crate::pr_head::owner_lock_wait(
1604            &crate::run::home(),
1605            &self.state.id,
1606            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607        )
1608        .await
1609        {
1610            Ok(owner) => owner,
1611            Err(error) => {
1612                let detail = format!("worktree maintenance or ownership update: {error:#}");
1613                // Admission did not own run.json: retain this resource outcome
1614                // in memory for queue settlement without overwriting a concurrent
1615                // retirement or publication transaction.
1616                self.state.resource_block = Some(crate::run::ResourceBlock {
1617                    cause: crate::run::ResourceBlockCause::Other,
1618                    node: "start".into(),
1619                    kind: "worktree maintenance".into(),
1620                    detail: detail.clone(),
1621                    at: Timestamp::now(),
1622                });
1623                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624            }
1625        };
1626        let disk = RunState::load(&self.state.id)?;
1627        crate::pr_head::ensure_owner(&disk)?;
1628        let claimed =
1629            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630        // Only proven-live ownership refuses here. Unknown records (legacy runs
1631        // with no claim marker or process identity) fall through to the
1632        // exclusive execution lease taken just below, which is what actually
1633        // keeps a second driver out.
1634        anyhow::ensure!(
1635            disk.liveness(claimed) != crate::run::Liveness::Live,
1636            "run already has a live driver; wait for its execution to end"
1637        );
1638        if let Some(to) = &disk.released_to {
1639            bail!(
1640                "run {} cannot continue: its worktree was released to run {}",
1641                self.state.short(),
1642                crate::run::short_of(to)
1643            );
1644        }
1645        anyhow::ensure!(
1646            serde_json::to_value(&disk.base_sync_recovery)?
1647                == serde_json::to_value(&self.state.base_sync_recovery)?,
1648            "base recovery changed since this run was loaded; reload the saved run"
1649        );
1650        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651        let mut execution = DriverExecution {
1652            runner: self,
1653            driver,
1654        };
1655        // Declare after the execution guard: on any claim error this short
1656        // lock must drop before that guard reloads and releases ownership.
1657        let _owner = owner;
1658        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659            .await?;
1660        let state = &mut execution.runner.state;
1661        // Moving again, so it is no longer parked. Set before the walk rather
1662        // than in `resume`, so every way of re-entering the graph clears it
1663        // and a card cannot claim a run is waiting to be resumed while the
1664        // agents are already working.
1665        state.parked = false;
1666        state.resource_wait = None;
1667        state.resource_block = None;
1668        // Any seat this state still lists as answering belongs to whatever
1669        // process last drove this run — this one included, if it crashed
1670        // mid-wave. Cleared and flushed immediately, before anything else
1671        // runs, so a resume can never show a seat as live when nothing is
1672        // asking it anything yet; the node that actually dispatches the next
1673        // wave repopulates it.
1674        state.clear_active();
1675        // Recorded in the same spot, and flushed together with the clear
1676        // above: this is the pid a reader checks (`RunState::liveness`) when
1677        // no daemon claim exists to answer "is a process still driving this
1678        // run" — a plain `magi run` / `magi review` typed into a terminal
1679        // claims nothing there. Always overwritten, never only-if-absent, so
1680        // a resumed run's stale pid from a previous, possibly-dead process
1681        // can never survive into this one's own report. Unlike
1682        // `clear_active`, this changes on every single `execute()` call, so
1683        // the save below is now unconditional rather than only-if-cleared.
1684        //
1685        // `driver_started_at` is recorded in the same breath, from this same
1686        // pid, so `liveness` can tell a live pid that is genuinely still us
1687        // apart from one the OS has since handed to an unrelated process —
1688        // see that field's own doc for why the pid alone is not enough.
1689        let pid = std::process::id();
1690        state.driver_pid = Some(pid);
1691        state.driver_started_at = crate::proc::process_started_at(pid);
1692        state.driver_exited = false;
1693        state.driver_claimed = Some(true);
1694        state.begin_pass();
1695        execution.driver.bind(state);
1696        state.save()?;
1697        execution.driver.record(state)?;
1698        Ok(execution)
1699    }
1700
1701    /// A grant owns only these repair attempts. Confirmation reviews cannot
1702    /// create another fixer allowance, and every operation is restart-checkpointed.
1703    async fn execute_granted_repair(&mut self) -> Result<()> {
1704        use crate::repair_budget::{Kind, Phase};
1705        let index = self.state.repair_grants.len() - 1;
1706        let winner = self
1707            .state
1708            .winner()
1709            .cloned()
1710            .context("repair winner missing")?;
1711        loop {
1712            let grant = self.state.repair_grants[index].clone();
1713            match grant.phase {
1714                Phase::Pending => {
1715                    if grant.spent >= grant.receipt.request.rounds {
1716                        self.state.repair_grants[index].phase = Phase::Complete;
1717                        self.state.status = RunStatus::Blocked;
1718                        self.state.save()?;
1719                        return Ok(());
1720                    }
1721                    // Reserve before dispatch. A crash can consume an attempt,
1722                    // but can never duplicate it or reset saved fixer history.
1723                    self.state.repair_grants[index].spent += 1;
1724                    self.state.repair_grants[index].phase = Phase::Fixing;
1725                    self.state.save()?;
1726                    match grant.receipt.request.kind {
1727                        Kind::Gate => {
1728                            let failed = self.state.gate.clone();
1729                            let _ = self.gate_fix_round(&winner, &failed).await?;
1730                        }
1731                        Kind::Review => {
1732                            let mut record = self
1733                                .state
1734                                .reviews
1735                                .last()
1736                                .cloned()
1737                                .context("review failure missing")?;
1738                            let round = record.round;
1739                            let step = self.run_fix_round(&winner, &mut record).await?;
1740                            self.state.reviews[round - 1] = record;
1741                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1742                                self.state.repair_grants[index].phase = Phase::Complete;
1743                                self.state.status = RunStatus::Blocked;
1744                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1745                                self.state.save()?;
1746                                return Ok(());
1747                            }
1748                            if matches!(step, FixStep::Resource) {
1749                                self.stop_for_resource()?;
1750                                return Ok(());
1751                            }
1752                            self.state.save()?;
1753                        }
1754                    }
1755                    if self.resource_halt()? {
1756                        return Ok(());
1757                    }
1758                }
1759                Phase::Fixing => {
1760                    // Interrupted fixers may have left source changes or an
1761                    // unpushed commit. Preserve them before fresh assessment.
1762                    let _coordination = self.coordinate(&winner.worktree).await?;
1763                    let rescued = git::rescue_commit(
1764                        &winner.worktree,
1765                        "magi: preserve interrupted granted repair",
1766                    )
1767                    .await?;
1768                    self.state.note_withheld("repair-grant", &rescued.withheld);
1769                    // Fresh checks must supersede even a same-head failure.
1770                    // Archive rather than erase failure evidence or counters.
1771                    self.state
1772                        .verification_history
1773                        .push(crate::run::VerificationEvidence {
1774                            superseded_by: None,
1775                            archived_at: Some(Timestamp::now()),
1776                            fixes_spent: Some(self.state.gate_fixes.len()),
1777                            gate_head: self.state.gate_head.take(),
1778                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1779                            gate: std::mem::take(&mut self.state.gate),
1780                            e2e: self.state.head_e2e.take(),
1781                        });
1782                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1783                        self.state
1784                            .verification_history
1785                            .push(crate::run::VerificationEvidence {
1786                                superseded_by: None,
1787                                archived_at: Some(Timestamp::now()),
1788                                fixes_spent: Some(self.state.gate_fixes.len()),
1789                                gate_head: None,
1790                                gate_ran: false,
1791                                gate: Vec::new(),
1792                                e2e: Some(e2e),
1793                            });
1794                    }
1795                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1796                    self.state.repair_grants[index].head =
1797                        git::rev_parse(&winner.worktree, "HEAD").await?;
1798                    self.state.repair_grants[index].phase = Phase::Reviewing;
1799                    self.state.save()?;
1800                }
1801                Phase::Reviewing => {
1802                    self.review_loop_pass().await?;
1803                    if self.resource_halt()? {
1804                        return Ok(());
1805                    }
1806                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1807                    let clean = self.state.reviews.len() > grant.review_start
1808                        && self.state.reviews.last().is_some_and(|r| {
1809                            r.head == head
1810                                && r.clean
1811                                && r.expected > 0
1812                                && r.answered == r.expected
1813                                && matches!(
1814                                    r.e2e_status(),
1815                                    E2eStatus::Passed | E2eStatus::NotConfigured
1816                                )
1817                        });
1818                    if !clean {
1819                        // A confirmation review carries no additional fix budget.
1820                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1821                            == Kind::Review
1822                            && grant.spent < grant.receipt.request.rounds
1823                            && self.state.reviews.last().is_some_and(|r| {
1824                                r.fix.is_none()
1825                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1826                            }) {
1827                            Phase::Pending
1828                        } else {
1829                            Phase::Complete
1830                        };
1831                        self.state.status = RunStatus::Blocked;
1832                        self.state.event(
1833                            "repair-grant",
1834                            "fresh review did not clear the repaired head; no publication",
1835                        );
1836                        self.state.save()?;
1837                        if self.state.repair_grants[index].phase == Phase::Pending {
1838                            continue;
1839                        }
1840                        return Ok(());
1841                    }
1842                    self.state.repair_grants[index].phase = Phase::Gating;
1843                    self.state.save()?;
1844                }
1845                Phase::Gating => {
1846                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1847                    self.state.reconcile_verification_head(&head);
1848                    let wait = self.repository_wait();
1849                    if self
1850                        .state
1851                        .reviews
1852                        .last()
1853                        .is_none_or(|r| r.head != head || !r.clean)
1854                    {
1855                        self.state.status = RunStatus::Blocked;
1856                        self.state.repair_grants[index].phase = Phase::Complete;
1857                        self.state.event(
1858                            "repair-grant",
1859                            "head changed since fresh review; repair held",
1860                        );
1861                        self.state.save()?;
1862                        return Ok(());
1863                    }
1864                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1865                        return Ok(());
1866                    }
1867                    let outcomes = self.run_gate(&winner).await?;
1868                    if verify_inconclusive(&outcomes) {
1869                        return Ok(());
1870                    }
1871                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1872                    if after != head {
1873                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1874                        self.state.repair_grants[index].head = after;
1875                        self.state.repair_grants[index].phase = Phase::Reviewing;
1876                        self.state.status = RunStatus::Blocked;
1877                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1878                        self.state.save()?;
1879                        return Ok(());
1880                    }
1881                    let passed = outcomes.iter().all(CommandOutcome::ok);
1882                    self.state.repair_grants[index].succeeded = passed;
1883                    self.state.gate = outcomes;
1884                    self.state.gate_ran = true;
1885                    self.state.status = if passed {
1886                        RunStatus::Gating
1887                    } else {
1888                        RunStatus::Blocked
1889                    };
1890                    self.state.repair_grants[index].phase = if !passed
1891                        && grant.receipt.request.kind == Kind::Gate
1892                        && grant.spent < grant.receipt.request.rounds
1893                    {
1894                        Phase::Pending
1895                    } else {
1896                        Phase::Complete
1897                    };
1898                    if passed && self.state.config.merge.mode == MergeMode::Pr {
1899                        // Persist publication debt with the successful checkpoint.
1900                        self.state.pr_publication_pending = true;
1901                    } else if passed {
1902                        self.state.status = RunStatus::Ready;
1903                    }
1904                    self.state.save()?;
1905                    if !passed {
1906                        continue;
1907                    }
1908                    // Existing PR ownership and push lease stay in place. The
1909                    // normal publisher owes the new reviewed/gated SHA, and
1910                    // land's fresh owner approval remains mandatory.
1911                    if self.state.config.merge.mode != MergeMode::Pr {
1912                        self.state.status = RunStatus::Ready;
1913                        self.state.event(
1914                            "repair-grant",
1915                            "repair verified locally; grant carries no merge authorization",
1916                        );
1917                        self.state.save()?;
1918                        return Ok(());
1919                    }
1920                    return self.merge().await;
1921                }
1922                Phase::Complete => return Ok(()),
1923            }
1924        }
1925    }
1926
1927    async fn execute_graph(&mut self) -> Result<()> {
1928        self.state.configure_recovery_resume()?;
1929        if self
1930            .state
1931            .repair_grants
1932            .last()
1933            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1934        {
1935            return self.execute_granted_repair().await;
1936        }
1937        if self
1938            .state
1939            .repair_grants
1940            .last()
1941            .is_some_and(|g| !g.succeeded)
1942        {
1943            self.state.status = RunStatus::Blocked;
1944            self.state.save()?;
1945            return Ok(());
1946        }
1947        // Preservation may invalidate an old empty classification. Resolve it
1948        // before any resume shortcut can reuse a verdict or publish a branch.
1949        self.persist_candidates().await?;
1950        // A saved base stop outranks stage markers, including legacy Voting.
1951        // Recovery never reenters implementation, deliberation or voting.
1952        if self.state.base_sync_blocked() {
1953            self.sync_to_base().await?;
1954            if !self.state.base_sync_blocked()
1955                && self.state.status != RunStatus::AlreadyInBase
1956                && !self.resource_halt()?
1957            {
1958                self.finish_after_tally().await?;
1959            }
1960            return Ok(());
1961        }
1962        if self
1963            .state
1964            .merge_decision
1965            .as_ref()
1966            .is_some_and(|decision| !decision.closed)
1967            && self.state.status != RunStatus::Landing
1968        {
1969            if crate::merge_decision::resumable_hold(&self.state) {
1970                self.state.merge = Some(MergeOutcome {
1971                    mode: MergeMode::Pr,
1972                    ok: false,
1973                    detail: self
1974                        .state
1975                        .merge_decision
1976                        .as_ref()
1977                        .unwrap()
1978                        .target
1979                        .pr
1980                        .clone(),
1981                    empty: false,
1982                });
1983                self.run_land().await?;
1984                self.settle_questions();
1985                return Ok(());
1986            }
1987            bail!(
1988                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
1989            );
1990        }
1991        if self.state.config.graph.selected_implementer.is_some()
1992            && self.state.status == RunStatus::Blocked
1993            && self.state.tally.is_none()
1994            && self.state.candidate_preservation.is_empty()
1995            && self.state.interrupted_implementations.is_empty()
1996        {
1997            bail!(
1998                "selected implementation is held; explicitly release the owning task to retry it"
1999            );
2000        }
2001
2002        if self.state.merge_repairs.last().is_some_and(|r| {
2003            matches!(
2004                r.phase,
2005                crate::run::MergeRepairPhase::Requested
2006                    | crate::run::MergeRepairPhase::Fixing
2007                    | crate::run::MergeRepairPhase::Reviewing
2008            )
2009        }) {
2010            if self.repair_merge_review().await? {
2011                self.run_land().await?;
2012            }
2013            self.settle_questions();
2014            return Ok(());
2015        }
2016        // Publication already passed review and gate. Resume only that node:
2017        // syncing to a newer base here could change the verified commit.
2018        if self.state.pr_publication_pending {
2019            self.merge().await?;
2020            if self.resource_halt()? {
2021                return Ok(());
2022            }
2023            self.state.save()?;
2024            self.settle_questions();
2025            return Ok(());
2026        }
2027        // A run that already lost its quorum never resumes into the verdict
2028        // machinery: `deliberate` and `vote` would otherwise clobber the
2029        // stalled marker back to Voting and the run would keep going past a
2030        // verdict that is no longer trustworthy. Everything already recorded is
2031        // kept, so the run stays resumable (or foldable) for a human to pick up.
2032        //
2033        // On --resume the run gets one chance to repair itself: the seats a
2034        // rate limit took out are re-asked. If their quota has since reset and
2035        // the quorum is restored, the run picks up and finishes; otherwise it
2036        // stays stale and still-resumable for a later retry. If it does not
2037        // recover, the returned status stays `Stalled` and nothing was
2038        // clobbered (the recovery only mutates entries for the lost seats).
2039        if self.state.status == RunStatus::Stalled {
2040            if self.recover_stall().await? {
2041                self.finish_after_tally().await?;
2042            } else {
2043                // Still below quorum: persist the marker and stay resumable.
2044                self.state.save()?;
2045            }
2046            return Ok(());
2047        }
2048        // A run parked inside `land` - watching CI, mid fix-round, or
2049        // waiting on the owner's merge approval - resumes directly into it,
2050        // never back through `prep`. Everything before `merge` already
2051        // concluded; that is the only way `status` reaches `Landing` in the
2052        // first place. Re-walking `review_loop` first would also be actively
2053        // wrong: its own status recomputation (see its doc) treats any
2054        // clean round as reason to set `status` to `Gating`, which would
2055        // clobber this marker before `merge` ever ran, and this run would
2056        // never find its way back into `land` at all.
2057        if self.state.status == RunStatus::Landing
2058            || crate::handover::landing_checkpoint(&self.state)
2059        {
2060            self.state.land_sync_pending = true;
2061            self.state.save()?;
2062            self.run_land().await?;
2063            // `run_land` may have settled the run right here - CI came back
2064            // green and the PR merged, say - without ever passing back
2065            // through `merge`'s own trailing call. Whatever it left `status`
2066            // as is what this has to read.
2067            self.settle_questions();
2068            return Ok(());
2069        }
2070        // A run parked at `merge` on the owner's word about a withheld PR text
2071        // resumes straight into `merge`. Re-walking the tail would sync to the
2072        // base again (possibly rebasing, possibly spending the fixer) after the
2073        // gate had passed, and then push a tree nobody reviewed or gated.
2074        if self.state.github_text.is_some()
2075            && self.state.merge.is_none()
2076            && self.state.status == RunStatus::Gating
2077        {
2078            self.merge().await?;
2079            if self.resource_halt()? {
2080                return Ok(());
2081            }
2082            self.state.save()?;
2083            return Ok(());
2084        }
2085        if let Err(e) = self.prep().await {
2086            if e.is::<crate::repo_lock::ResourceBlocked>() {
2087                return Err(e);
2088            }
2089            self.state.status = RunStatus::Blocked;
2090            self.state
2091                .prep_failure
2092                .get_or_insert_with(|| format!("{e:#}"));
2093            self.state
2094                .event("prep", format!("preparation stopped: {e:#}"));
2095            self.state.save()?;
2096            return Err(e);
2097        }
2098        if self.park_here()? {
2099            return Ok(());
2100        }
2101        self.advise().await?;
2102        if self.park_here()? {
2103            return Ok(());
2104        }
2105        self.implement().await?;
2106        if self.park_here()? {
2107            return Ok(());
2108        }
2109        // `after_implement` already saved the state and settled any open
2110        // questions when it set this; nothing later in the graph has
2111        // anything to judge.
2112        if self.state.status == RunStatus::VerifiedNoop {
2113            return Ok(());
2114        }
2115        self.judge().await?;
2116        if self.park_here()? {
2117            return Ok(());
2118        }
2119        self.deliberate().await?;
2120        if self.park_here()? {
2121            return Ok(());
2122        }
2123        self.vote().await?;
2124        if self.park_here()? {
2125            return Ok(());
2126        }
2127        self.tally()?;
2128        // A verdict that lost its quorum is not trustworthy: do not review,
2129        // gate, or merge on it. Everything already done is kept, so the run
2130        // stays resumable (or foldable); the human can replace the agent that
2131        // ran out of quota and pick it up.
2132        if self.state.status == RunStatus::Stalled {
2133            // Persist the stalled marker now — the normal end-of-execute save
2134            // below is below this early return, and without it a resumed run
2135            // would reload a pre-tally status and keep going.
2136            self.state.save()?;
2137            return Ok(());
2138        }
2139        self.finish_after_tally().await?;
2140        Ok(())
2141    }
2142
2143    /// A wave could not start its agents because a shared resource stayed busy
2144    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2145    /// resumable. Unstarted seats have no result; completed peers retain
2146    /// their saved candidate state.
2147    fn stop_for_resource(&mut self) -> Result<()> {
2148        // Resource acquisition observes cancellation inside a node. Complete
2149        // the same durable park as every other node boundary before returning.
2150        if self.pause.parked() || self.interrupt.parked() {
2151            self.park_here()?;
2152            return Ok(());
2153        }
2154        if self.state.resource_block.is_some() {
2155            self.state.status = RunStatus::Blocked;
2156        }
2157        self.state.save()
2158    }
2159
2160    /// Stop walking the graph if a node ended on a resource block.
2161    fn resource_halt(&mut self) -> Result<bool> {
2162        if self.state.resource_block.is_none() {
2163            return Ok(false);
2164        }
2165        self.stop_for_resource()?;
2166        Ok(true)
2167    }
2168
2169    /// Park here if asked to, recording it in the run's own timeline.
2170    ///
2171    /// Returns whether the caller should stop walking the graph. The state is
2172    /// saved either way by the node that just finished; this adds the event so
2173    /// the operator's card says why a run that is neither finished nor moving
2174    /// is sitting where it is.
2175    fn park_here(&mut self) -> Result<bool> {
2176        let parking = self.pause.parked() || self.interrupt.parked();
2177        if !parking && self.state.resource_block.is_some() {
2178            self.stop_for_resource()?;
2179            return Ok(true);
2180        }
2181        // Either handle asking is enough - see `Pause`'s own doc for why
2182        // they are never the same one. `interrupt` is checked second so a
2183        // reason it carries is preferred in the message below over a plain
2184        // shutdown park racing it at the same boundary.
2185        if !self.pause.parked() && !self.interrupt.parked() {
2186            return Ok(false);
2187        }
2188        if self.state.parked {
2189            return Ok(true);
2190        }
2191        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2192        if let Some(block) = &mut self.state.resource_block
2193            && matches!(
2194                block.cause,
2195                crate::run::ResourceBlockCause::Cancelled { .. }
2196            )
2197        {
2198            block.cause = crate::run::ResourceBlockCause::Cancelled {
2199                park: cause.clone(),
2200            };
2201        }
2202        let why = match cause.as_ref().map(ParkCause::label) {
2203            Some(reason) => format!(
2204                "parked after `{}` ({reason}) — resume to carry on from here",
2205                self.state.status.as_str()
2206            ),
2207            None => format!(
2208                "parked after `{}` — resume to carry on from here",
2209                self.state.status.as_str()
2210            ),
2211        };
2212        self.state.event("park", why);
2213        self.state
2214            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2215        self.state.parked = true;
2216        self.state.save()?;
2217        Ok(true)
2218    }
2219
2220    /// Only a new explicit operator approval may retry failed preparation.
2221    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2222        if self.state.prep_failure.is_none() {
2223            return Ok(());
2224        }
2225        let failed_at = self
2226            .state
2227            .events
2228            .iter()
2229            .rev()
2230            .find(|event| event.node == "prep")
2231            .map_or(self.state.updated_at, |event| event.at);
2232        anyhow::ensure!(
2233            approval_at > failed_at,
2234            "preparation retry requires a new explicit release after the failure"
2235        );
2236        self.state.prep_failure = None;
2237        self.state.prep_complete = Some(false);
2238        self.state
2239            .event("prep-retry", "operator approved retry of preparation");
2240        Ok(())
2241    }
2242
2243    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2244    pub fn on_pause(&mut self, pause: Pause) {
2245        self.pause = pause;
2246    }
2247
2248    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2249    /// scheduler asking this one run - and no other - to park so a task
2250    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2251    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2252    /// handle.
2253    pub fn watch_interrupt(&mut self, pause: Pause) {
2254        self.interrupt = pause;
2255    }
2256
2257    /// Abandon this run's own open questions, once `status` has actually
2258    /// settled rather than merely paused.
2259    ///
2260    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2261    /// either back up with the candidates, the review round and the seat
2262    /// sessions already on disk, so a question an implementer asked mid-round
2263    /// may still get a real answer read by a real resume. Only the statuses
2264    /// `resumable` excludes are actually final: the run merged, it reached
2265    /// `Ready` with nothing left to do, it failed outright with no
2266    /// established point to continue from, or every candidate agreed, with
2267    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2268    /// every one of those the seat that asked is gone for good, exactly like
2269    /// the run being deleted under `magi run rm` - so the same cleanup
2270    /// applies, worded for what actually happened instead of "the run was
2271    /// deleted".
2272    ///
2273    /// Best-effort and silent on success: called from every place `status`
2274    /// can land on one of those three, including ones a resumed run revisits,
2275    /// so it must cost nothing when there was nothing open to begin with.
2276    fn settle_questions(&mut self) {
2277        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2278            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2279        }
2280    }
2281
2282    /// The tail of the graph after a trustworthy tally: fold losers, review,
2283    /// gate, merge, and persist.
2284    async fn finish_after_tally(&mut self) -> Result<()> {
2285        // Before the losers are folded: their branches are what the graft
2286        // reads from.
2287        self.graft().await?;
2288        if self.resource_halt()? {
2289            return Ok(());
2290        }
2291        self.fold_losers().await?;
2292        // Before review starts, and again right before the gate: a run's
2293        // review rounds can themselves take long enough for the base to move
2294        // a second time, and the gate is the one node whose "green" gets
2295        // acted on.
2296        self.sync_to_base().await?;
2297        if self.hold_base_sync().await? || self.resource_halt()? {
2298            return Ok(());
2299        }
2300        if self.state.status == RunStatus::AlreadyInBase {
2301            return Ok(());
2302        }
2303        self.review_loop().await?;
2304        if self.resource_halt()? {
2305            return Ok(());
2306        }
2307        self.sync_to_base().await?;
2308        if self.hold_base_sync().await? || self.resource_halt()? {
2309            return Ok(());
2310        }
2311        if self.state.status == RunStatus::AlreadyInBase {
2312            return Ok(());
2313        }
2314        self.gate().await?;
2315        self.merge().await?;
2316        if self.resource_halt()? {
2317            return Ok(());
2318        }
2319        self.state.save()?;
2320        Ok(())
2321    }
2322
2323    // ---------------------------------------------------------------- prep
2324
2325    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2326    /// run with the step and its output in the event; it is never swallowed.
2327    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2328    /// resume starts over instead of finding candidates and missing seats.
2329    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2330        let result = match ensure_setup_config(&mut self.state, repo).await {
2331            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2332            Err(e) => Err(e),
2333        };
2334        if let Err(e) = result {
2335            git::worktree_remove_clean(repo, wt).await.ok();
2336            if in_prep {
2337                self.rollback_prep(repo).await;
2338            }
2339            self.state.status = RunStatus::Blocked;
2340            self.state.event(
2341                "setup",
2342                format!("worktree setup failed in {}: {e:#}", wt.display()),
2343            );
2344            self.state.save()?;
2345            return Err(e);
2346        }
2347        Ok(())
2348    }
2349
2350    /// Undo everything `prep` created, so a resume re-runs it whole.
2351    async fn rollback_prep(&mut self, repo: &Path) {
2352        let root = self.state.worktree_root();
2353        for c in std::mem::take(&mut self.state.candidates) {
2354            if matches!(
2355                git::worktree_remove_clean(repo, &c.worktree).await,
2356                Ok(true)
2357            ) {
2358                git::branch_delete(repo, &c.branch).await.ok();
2359            } else {
2360                self.state.candidates.push(c);
2361            }
2362        }
2363        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2364        for j in 1..=seats {
2365            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2366                .await
2367                .ok();
2368        }
2369        for k in 1..=self.state.config.graph.advisors {
2370            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2371                .await
2372                .ok();
2373        }
2374    }
2375
2376    async fn prep(&mut self) -> Result<()> {
2377        if let Some(reason) = &self.state.prep_failure {
2378            bail!(
2379                "candidate preparation previously failed: {reason}; correct the source before retrying"
2380            );
2381        }
2382        if self
2383            .state
2384            .prep_complete
2385            .unwrap_or(!self.state.candidates.is_empty())
2386        {
2387            return Ok(());
2388        }
2389        let repo = self.state.repo.clone();
2390        // Includes rollback, config, refs, checkout creation and setup. No
2391        // repository mutation may precede the real shared lock acquisition.
2392        let _coordination = self.coordinate(&repo).await?;
2393        let base = self.state.base_commit.clone();
2394        let plan = self.state.sources.plan(&repo, &base).await?;
2395        self.rollback_prep(&repo).await;
2396        if !self.state.candidates.is_empty() {
2397            bail!(
2398                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2399            );
2400        }
2401        self.state.prep_complete = Some(false);
2402        self.state.status = RunStatus::Prep;
2403        self.state.save()?;
2404        let repo = self.state.repo.clone();
2405        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2406        let root = self.state.worktree_root();
2407        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2408
2409        // The hook is the write-time half of the blindness contract; the
2410        // presentation filter in `blind` is the half that cannot be bypassed.
2411        let hooks_dir = self.state.dir().join("hooks");
2412        if self.state.config.blind.commit_msg_hook {
2413            std::fs::create_dir_all(&hooks_dir)
2414                .with_context(|| format!("create {}", hooks_dir.display()))?;
2415            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2416            let path = hooks_dir.join("commit-msg");
2417            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2418            make_executable(&path)?;
2419            // Ref-counted rather than a plain idempotent set: with more than
2420            // one run able to be in flight in the same repository at once
2421            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2422            // true?" check cannot tell "another run of mine still needs
2423            // this" from "nobody does", and the run that happens to finish
2424            // first would disable the hook out from under a sibling still
2425            // relying on it.
2426            git::acquire_worktree_config(&repo).await?;
2427            self.state.enabled_worktree_config = true;
2428        }
2429
2430        // `[worktree] setup` hides its products per worktree, which needs the
2431        // same per-worktree config the hook does. Held until fold, like it.
2432        ensure_setup_config(&mut self.state, &repo).await?;
2433
2434        for (index, (spec, label)) in self
2435            .roles
2436            .implementers
2437            .clone()
2438            .into_iter()
2439            .zip(labels)
2440            .enumerate()
2441        {
2442            let branch = self.state.branch_for(label);
2443            let worktree = root.join(format!("cand-{label}"));
2444            // Persist intent before Git can leave a partially created tree.
2445            // Candidate registration follows setup, so it cannot inventory
2446            // crashes or refusals during creation itself.
2447            self.state.event(
2448                "prep-intent",
2449                serde_json::json!({
2450                    "branch": branch,
2451                    "worktree": worktree,
2452                    "start": start,
2453                    "picks": plan.picks,
2454                })
2455                .to_string(),
2456            );
2457            self.state.save()?;
2458            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2459            if self.state.config.blind.commit_msg_hook {
2460                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2461            }
2462            git::local_exclude(&worktree, "/.magi/").await?;
2463            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2464                // Nothing of this prep may survive: a resume must start over,
2465                // not find half the candidates and skip the rest.
2466                git::worktree_remove_clean(&repo, &worktree).await.ok();
2467                git::branch_delete(&repo, &branch).await.ok();
2468                self.rollback_prep(&repo).await;
2469                self.state.status = RunStatus::Blocked;
2470                self.state
2471                    .event("prep", format!("worktree setup failed: {e:#}"));
2472                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2473                self.state.save()?;
2474                return Err(e);
2475            }
2476            self.state.candidates.push(Candidate {
2477                index,
2478                label,
2479                agent: spec.id.clone(),
2480                branch,
2481                worktree,
2482                summary: String::new(),
2483                stat: String::new(),
2484                files: 0,
2485                commits: 0,
2486                empty: false,
2487                failed: None,
2488                verified_noop: None,
2489                duration_ms: 0,
2490                folded: false,
2491            });
2492            self.state.save()?;
2493            for pick in &plan.picks {
2494                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2495                {
2496                    self.state.status = RunStatus::Blocked;
2497                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2498                    self.state
2499                        .event("prep", format!("cannot apply referenced commit: {e}"));
2500                    self.state.save()?;
2501                    return Err(e);
2502                }
2503            }
2504        }
2505
2506        for j in 1..=self.roles.judges.len() {
2507            let wt = root.join(format!("judge-{j}"));
2508            if !wt.exists() {
2509                git::worktree_add_detached(&repo, &wt, &base).await?;
2510                self.setup_seat_worktree(&repo, &wt, true).await?;
2511            }
2512        }
2513
2514        // Disposable, detached checkouts for the design-deliberation stage's
2515        // advisor seats — the same shape as the judges' above, at the same
2516        // base commit, since advisors also only ever read. Sized off the
2517        // configured count directly rather than a resolved roster: unlike
2518        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2519        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2520        // `prep` has no `ResolvedRoles` field to read a count from here.
2521        if self.state.config.graph.advise {
2522            for k in 1..=self.state.config.graph.advisors {
2523                let wt = root.join(format!("advisor-{k}"));
2524                if !wt.exists() {
2525                    git::worktree_add_detached(&repo, &wt, &base).await?;
2526                    self.setup_seat_worktree(&repo, &wt, true).await?;
2527                }
2528            }
2529        }
2530
2531        // A judge cannot tell it is looking at its own patch — the seats keep
2532        // separate conversations — but a panel that shares agents with the
2533        // field is less independent than it looks, and that is worth saying out
2534        // loud once per run rather than leaving it in the config.
2535        let authors: Vec<&str> = self
2536            .roles
2537            .implementers
2538            .iter()
2539            .map(|a| a.id.as_str())
2540            .collect();
2541        let overlap: Vec<String> = self
2542            .roles
2543            .judges
2544            .iter()
2545            .enumerate()
2546            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2547            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2548            .collect();
2549        if !overlap.is_empty() {
2550            let note = format!(
2551                "{} also authored a candidate; blind, but the panel is less \
2552                 independent than {} distinct agents would be",
2553                overlap.join(", "),
2554                self.roles.judges.len()
2555            );
2556            self.state.event("prep", note);
2557        }
2558
2559        self.state.event(
2560            "prep",
2561            format!(
2562                "{} candidates, {} judges, base {} ({})",
2563                self.state.candidates.len(),
2564                self.roles.judges.len(),
2565                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2566                self.state.base_branch
2567            ),
2568        );
2569        self.state.prep_complete = Some(true);
2570        self.state.status = RunStatus::Implementing;
2571        self.state.save()?;
2572        Ok(())
2573    }
2574
2575    // -------------------------------------------------------------- advise
2576
2577    /// The design-deliberation stage: independent, read-only advisor seats
2578    /// each sketch a design before any implementer touches the repository,
2579    /// and (when at least one produced a usable proposal) a synthesis seat
2580    /// blends them into a brief `implement` carries in every candidate's
2581    /// prompt.
2582    ///
2583    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2584    /// advisors` is the proposal count. Everything here is best-effort and
2585    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2586    /// that cannot reach quota, or a synthesis seat that produced nothing
2587    /// usable all leave `implement` exactly as it was before this stage
2588    /// existed — the task instruction alone — rather than failing the whole
2589    /// competition over an enrichment stage. Every outcome is still recorded
2590    /// as an event, so a run that got nothing from this stage says why.
2591    ///
2592    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2593    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2594    /// resumed run whose stage failed would re-run it, and re-spend the
2595    /// agent calls, on every reentry before `implement`.
2596    ///
2597    /// Also skipped once any candidate shows implementation progress — the
2598    /// exact predicate `implement` itself uses to decide a candidate is no
2599    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2600    /// is not enough: a run created by an older binary that predates this
2601    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2602    /// an already-`Implementing`-or-later run under this build would
2603    /// otherwise walk straight back through `prep` (a no-op once candidates
2604    /// exist) into this node and spawn every advisor seat against worktrees
2605    /// `prep` never recreated — after implementation has already started,
2606    /// which is exactly the invariant this stage exists to guarantee.
2607    async fn advise(&mut self) -> Result<()> {
2608        let implement_untouched = self
2609            .state
2610            .candidates
2611            .iter()
2612            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2613        if !self.state.config.graph.advise || self.state.advise_attempted {
2614            return Ok(());
2615        }
2616        if !implement_untouched {
2617            self.state.event(
2618                "advise",
2619                "skipping the design-deliberation stage: at least one \
2620                 candidate already shows implementation progress, so this \
2621                 run is past the point the stage exists to run before"
2622                    .to_owned(),
2623            );
2624            self.state.advise_attempted = true;
2625            self.state.save()?;
2626            return Ok(());
2627        }
2628        let run_id = self.state.id.clone();
2629        let prompts = self.state.config.prompts.clone();
2630        let instruction = self.state.instruction.clone();
2631        let language = self.state.config.graph.language.clone();
2632        let root = self.state.worktree_root();
2633        let n = self.state.config.graph.advisors;
2634        let where_recorded = self.state.dir().join("run.json");
2635
2636        let seats = match self.state.config.advisors() {
2637            Ok(seats) if !seats.is_empty() => seats,
2638            Ok(_) => {
2639                self.state.event(
2640                    "advise",
2641                    format!(
2642                        "[graph] advisors is 0; skipping the design-deliberation \
2643                         stage and continuing without a synthesis brief (see {})",
2644                        where_recorded.display()
2645                    ),
2646                );
2647                self.state.advise_attempted = true;
2648                self.state.save()?;
2649                return Ok(());
2650            }
2651            Err(e) => {
2652                self.state.event(
2653                    "advise",
2654                    format!(
2655                        "could not resolve advisor seats ({e:#}); continuing \
2656                         without a design-deliberation brief (see {})",
2657                        where_recorded.display()
2658                    ),
2659                );
2660                self.state.advise_attempted = true;
2661                self.state.save()?;
2662                return Ok(());
2663            }
2664        };
2665
2666        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2667        let artifacts = agent::artifacts_dir(&self.state.dir());
2668        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2669
2670        let mut jobs = Vec::new();
2671        for (i, spec) in seats.iter().cloned().enumerate() {
2672            let seat_key = format!("advisor-{}", i + 1);
2673            let seat = self.seat(&seat_key, &spec.id);
2674            jobs.push(SeatJob {
2675                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2676                spec,
2677                seat,
2678                cwd: worktrees[i % worktrees.len()].clone(),
2679                timeout,
2680                allow_write: false,
2681                sessions: false,
2682                artifacts: artifacts.clone(),
2683                stem: seat_key,
2684                handover: None,
2685            });
2686        }
2687
2688        self.state.event(
2689            "advise",
2690            format!(
2691                "{} advisor seat(s) sketching a design in parallel",
2692                jobs.len()
2693            ),
2694        );
2695        let mut quota_losses = Vec::new();
2696        let cache = self.state.config.cache_dir();
2697        let ctx = WaveCtx {
2698            carry_seats: false,
2699            pauses: Some((&self.pause, &self.interrupt)),
2700            run: &run_id,
2701            node: "advise",
2702            prompts: &prompts,
2703            cache: cache.as_deref(),
2704            round: None,
2705        };
2706        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2707        let results = ask_json_wave::<Proposal>(
2708            jobs,
2709            Arc::clone(&self.sem),
2710            self.state.config.graph.retries,
2711            &advisor_roster,
2712            &ctx,
2713            &mut quota_losses,
2714            &mut self.state,
2715            &|p: &Proposal| p.validate(),
2716        )
2717        .await;
2718        if self.state.resource_block.is_some() {
2719            return self.stop_for_resource();
2720        }
2721        self.state.quota.extend(quota_losses);
2722
2723        let mut records = Vec::with_capacity(results.len());
2724        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2725            let agent_id = seat.agent.clone();
2726            self.state.seats.insert(seat.key.clone(), seat);
2727            match res {
2728                Ok((proposal, out)) => {
2729                    self.state
2730                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2731                    records.push(advise::AdvisorRecord::proposed(
2732                        i + 1,
2733                        agent_id,
2734                        proposal,
2735                        out.duration_ms,
2736                    ));
2737                }
2738                Err(e) => {
2739                    self.state.event(
2740                        "advise",
2741                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2742                    );
2743                    records.push(advise::AdvisorRecord::failed(
2744                        i + 1,
2745                        agent_id,
2746                        e.to_string(),
2747                    ));
2748                }
2749            }
2750        }
2751
2752        let mut advice = advise::Advice {
2753            records,
2754            synthesis: None,
2755        };
2756        if advice.proposals().is_empty() {
2757            self.state.event(
2758                "advise",
2759                "no advisor produced a usable proposal; continuing without a \
2760                 synthesis brief"
2761                    .to_owned(),
2762            );
2763        } else {
2764            match self
2765                .synthesize_brief(
2766                    &advice,
2767                    &instruction,
2768                    &language,
2769                    &worktrees[0],
2770                    &artifacts,
2771                    &run_id,
2772                    &prompts,
2773                    cache.as_deref(),
2774                )
2775                .await
2776            {
2777                Ok(Some(text)) => {
2778                    self.state.event(
2779                        "advise",
2780                        "synthesized a design brief for the implementer".to_owned(),
2781                    );
2782                    advice.synthesis = Some(text);
2783                }
2784                Ok(None) => {
2785                    self.state.event(
2786                        "advise",
2787                        "the synthesis seat produced nothing usable; continuing \
2788                         without a design brief"
2789                            .to_owned(),
2790                    );
2791                }
2792                Err(e) => {
2793                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2794                        return Err(e);
2795                    }
2796                    self.state.event(
2797                        "advise",
2798                        format!("could not synthesize a design brief: {e:#}"),
2799                    );
2800                }
2801            }
2802        }
2803        advise::apply_reflection(&mut advice);
2804
2805        self.state.advice = Some(advice);
2806        self.state.advise_attempted = true;
2807        self.state.save()?;
2808        Ok(())
2809    }
2810
2811    /// The synthesis seat: reads every advisor's proposal and blends them
2812    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2813    /// out of [`Runner::advise`] only for readability — it is not called
2814    /// anywhere else.
2815    ///
2816    /// Picked the same way [`crate::talk`]'s standing conversation and
2817    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2818    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2819    /// order (a claude seat, else the first runnable agent in roster order)
2820    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2821    /// in [`crate::config`] for why a dedicated field exists here at all.
2822    #[allow(clippy::too_many_arguments)]
2823    async fn synthesize_brief(
2824        &mut self,
2825        advice: &advise::Advice,
2826        instruction: &str,
2827        language: &str,
2828        cwd: &Path,
2829        artifacts: &Path,
2830        run_id: &str,
2831        prompts: &Prompts,
2832        cache: Option<&Path>,
2833    ) -> Result<Option<String>> {
2834        let chain = agent::pick_chain(
2835            &self.state.config.agents,
2836            self.state.config.roles.synthesizer.as_ref(),
2837            &agent::installed,
2838            "synthesizer",
2839        )?;
2840        let proposals = advice.proposals();
2841        let mut prompt = prompt::with_overlay(
2842            prompt::synthesize_brief(instruction, &proposals, language),
2843            prompts.overlay("advise"),
2844        );
2845        if cache.is_some() {
2846            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2847            // below — see `prompt::build_cache_note`'s doc for why telling a
2848            // read-only seat to build through the shared cache is exactly how
2849            // a sandbox's write refusal gets misread as a defect.
2850            prompt.push('\n');
2851            prompt.push_str(&prompt::build_cache_note("advise", false));
2852        }
2853        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2854        // Each id is tried once, in order; a quota hit, error or unusable
2855        // answer moves to the next. The seat is single-turn (`sessions:
2856        // false`) and the prompt is the whole context, so a fallback agent
2857        // needs nothing carried over.
2858        let mut last = None;
2859        for (n, spec) in chain.iter().enumerate() {
2860            if n > 0 {
2861                self.state
2862                    .event("advise", format!("synthesis falling back to {}", spec.id));
2863            }
2864            let mut seat = self.seat("advise-synthesis", &spec.id);
2865            let wait = self.repository_wait();
2866            let outcome = wait
2867                .scope(agent::invoke(
2868                    spec,
2869                    &mut seat,
2870                    &Invocation {
2871                        coordination: self.state.config.coordination.as_ref(),
2872                        fetched_base: None,
2873                        cwd,
2874                        prompt: &prompt,
2875                        timeout,
2876                        allow_write: false,
2877                        sessions: false,
2878                        artifacts,
2879                        stem: &if n == 0 {
2880                            "advise-synthesis".to_owned()
2881                        } else {
2882                            format!("advise-synthesis-{}", spec.id)
2883                        },
2884                        run: run_id,
2885                        node: "advise",
2886                        cache_dir: None,
2887                        attachments: &[],
2888                        writable: &[],
2889                    },
2890                ))
2891                .await;
2892            if outcome.is_ok() {
2893                self.state.seats.insert(seat.key.clone(), seat);
2894            }
2895            let advance = agent::chain_advances(&outcome);
2896            last = Some(outcome);
2897            if !advance {
2898                break;
2899            }
2900        }
2901        // Exhausted: the last attempt's result is what a single failed seat
2902        // would have produced.
2903        let out = last.expect("a chain holds at least one agent")?;
2904        if !out.usable() {
2905            return Ok(None);
2906        }
2907        let text =
2908            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2909        Ok((!text.trim().is_empty()).then_some(text))
2910    }
2911
2912    // ----------------------------------------------------------- implement
2913
2914    async fn implement(&mut self) -> Result<()> {
2915        self.persist_candidates().await?;
2916        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2917        // agent files with `magi task add` name the run that paid for it. The
2918        // prompt overlay is cloned alongside it because the waves borrow it
2919        // while `self` is mutably borrowed by the node's own bookkeeping.
2920        let run_id = self.state.id.clone();
2921        let prompts = self.state.config.prompts.clone();
2922        let todo: Vec<usize> = self
2923            .state
2924            .candidates
2925            .iter()
2926            .enumerate()
2927            .filter(|(_, c)| {
2928                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2929                    && c.failed.is_none()
2930                    && !c.empty
2931            })
2932            .map(|(i, _)| i)
2933            .collect();
2934        if todo.is_empty() {
2935            return self.after_implement();
2936        }
2937        self.state.status = RunStatus::Implementing;
2938
2939        let language = self.state.config.graph.language.clone();
2940        let github_language = self.state.config.graph.github_language.clone();
2941        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
2942        let sessions = self.state.config.graph.sessions;
2943        let artifacts = agent::artifacts_dir(&self.state.dir());
2944        // The design-deliberation stage's blended brief, when `advise` found
2945        // one — carried into every implementer's prompt the same way
2946        // regardless of which candidate it is.
2947        let brief = self
2948            .state
2949            .advice
2950            .as_ref()
2951            .and_then(|a| a.synthesis.as_deref())
2952            .map(str::to_owned);
2953        let attachments = self.state.attachments.clone();
2954
2955        let mut jobs = Vec::new();
2956        for &i in &todo {
2957            let (index, label, worktree) = {
2958                let c = &self.state.candidates[i];
2959                (c.index, c.label, c.worktree.clone())
2960            };
2961            let spec = self.roles.implementers[index].clone();
2962            let seat_key = format!("impl-{label}");
2963            let seat = self.seat(&seat_key, &spec.id);
2964            let instruction = seeded_instruction(&self.state);
2965            jobs.push(SeatJob {
2966                spec,
2967                seat,
2968                prompt: prompt::implement_in(
2969                    &instruction,
2970                    &worktree.to_string_lossy(),
2971                    &language,
2972                    &github_language,
2973                    brief.as_deref(),
2974                    &attachments,
2975                ),
2976                cwd: worktree,
2977                timeout,
2978                allow_write: true,
2979                sessions,
2980                artifacts: artifacts.clone(),
2981                stem: format!("impl-{label}"),
2982                handover: None,
2983            });
2984        }
2985
2986        self.state.event(
2987            "implement",
2988            format!("{} candidates in parallel", jobs.len()),
2989        );
2990        // Kept so a seat whose CLI hung up can be asked again from the same
2991        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
2992        // can update a seat's own entry once a fallback agent takes it over —
2993        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
2994        // whichever agent actually answered, not the one that quota'd out.
2995        let mut sent = jobs.clone();
2996        let cache = self.state.config.cache_dir();
2997        let ctx = WaveCtx {
2998            carry_seats: false,
2999            pauses: Some((&self.pause, &self.interrupt)),
3000            run: &run_id,
3001            node: "implement",
3002            prompts: &prompts,
3003            cache: cache.as_deref(),
3004            round: None,
3005        };
3006        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3007        if !wave_blocked(&results) {
3008            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3009                .await;
3010            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3011                .await?;
3012            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3013                .await;
3014        }
3015        // Do not finalize failed candidates when the entire wave needs machine
3016        // recovery: leaving them pending resumes this node with its work intact.
3017        if !results.is_empty()
3018            && results
3019                .iter()
3020                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3021        {
3022            let losses: Vec<_> = results
3023                .iter()
3024                .map(|(_, seat, out)| {
3025                    let AgentOutcome::Quota(o) = out else {
3026                        unreachable!()
3027                    };
3028                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3029                })
3030                .collect();
3031            self.state.quota.extend(losses.clone());
3032            for (_, seat, _) in results {
3033                self.state.seats.insert(seat.key.clone(), seat);
3034            }
3035            for &i in &todo {
3036                let label = self.state.candidates[i].label;
3037                self.state.interrupted_implementations.insert(label);
3038                self.state
3039                    .candidate_preservation
3040                    .insert(label, "machine outage awaiting persistence".into());
3041            }
3042            // Preserve partial edits too, while keeping every interrupted seat
3043            // pending even when its tree already contains an inherited patch.
3044            block_machine_outage(&mut self.state, "implement", &losses);
3045            self.state.save()?;
3046            self.persist_candidates().await?;
3047            for &i in &todo {
3048                self.state
3049                    .interrupted_implementations
3050                    .insert(self.state.candidates[i].label);
3051            }
3052            return self.stop_for_resource();
3053        }
3054        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3055
3056        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3057            && results
3058                .iter()
3059                .any(|(_, _, out)| FailClass::of(out).is_some());
3060        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3061            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3062                // Executed turns interrupted by disk pressure already carry
3063                // a preservation barrier from wave. Admission-only waits do
3064                // not. Keep the interrupted seat's session for recovery too.
3065                if self
3066                    .state
3067                    .candidate_preservation
3068                    .contains_key(&self.state.candidates[i].label)
3069                {
3070                    self.state.seats.insert(seat.key.clone(), seat);
3071                }
3072                continue;
3073            }
3074            let seat_key = seat.key.clone();
3075            // A quota fallback (`resume_seat_handovers`) may have handed this
3076            // seat to a different agent than the one `prep` recorded on the
3077            // candidate; the stats tables and any later fixer-defaults-to-
3078            // winner's-author lookup must credit whoever actually answered —
3079            // unless every fallback also quota'd out, in which case nobody
3080            // actually answered and crediting the last agent tried would
3081            // erase every earlier agent's own quota loss from the stats
3082            // tables instead of just this one seat's.
3083            let agent = seat.agent.clone();
3084            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3085            self.state.seats.insert(seat.key.clone(), seat);
3086            let label = self.state.candidates[i].label;
3087            self.state.interrupted_implementations.remove(&label);
3088
3089            let (summary, duration, failed, verified_claim) = match out {
3090                AgentOutcome::Ok(o) => {
3091                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3092                    let failed = (!o.usable()).then(|| {
3093                        if o.timed_out {
3094                            "agent timed out".to_owned()
3095                        } else {
3096                            format!(
3097                                "agent exited with {}",
3098                                crate::notices::exit_status(o.exit_code)
3099                            )
3100                        }
3101                    });
3102                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3103                    (text, o.duration_ms, failed, verified_claim)
3104                }
3105                // Left un-resumed by `resume_undelivered` (a dirty tree
3106                // already rescues the work, or there was no session left to
3107                // resume into) — reported like the ordinary failure it is,
3108                // never as if `o.text` (the CLI's raw error JSON) were an
3109                // answer.
3110                AgentOutcome::Dropped(o) => {
3111                    let why = o
3112                        .dropped
3113                        .as_ref()
3114                        .map(|d| d.why.as_str())
3115                        .unwrap_or("the CLI ended the stream without delivering its answer");
3116                    (
3117                        String::new(),
3118                        o.duration_ms,
3119                        Some(format!("the CLI dropped the stream ({why})")),
3120                        None,
3121                    )
3122                }
3123                AgentOutcome::Quota(o) => {
3124                    self.state.quota.push(QuotaLoss::from_report(
3125                        seat_key,
3126                        "implement",
3127                        o.quota.as_ref(),
3128                    ));
3129                    (
3130                        String::new(),
3131                        o.duration_ms,
3132                        Some("rate limited (quota); produced no change".to_owned()),
3133                        None,
3134                    )
3135                }
3136                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3137                    (String::new(), 0, Some(e), None)
3138                }
3139            };
3140
3141            // Save every completed turn before any fallible Git operation. A failed
3142            // rescue must not erase the reply or prevent collection of other seats.
3143            let c = &mut self.state.candidates[i];
3144            if !exhausted_the_fallback_chain {
3145                c.agent = agent;
3146            }
3147            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3148            c.duration_ms = duration;
3149            c.failed = failed;
3150            c.verified_noop = verified_claim;
3151            self.state
3152                .candidate_preservation
3153                .insert(label, "awaiting persistence".to_owned());
3154        }
3155        self.state.save()?;
3156        self.persist_candidates().await?;
3157
3158        if resource_blocked {
3159            return self.stop_for_resource();
3160        }
3161        if selected_failed {
3162            self.state.status = RunStatus::Blocked;
3163            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3164            self.state.save()?;
3165            bail!(
3166                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3167            );
3168        }
3169        self.after_implement()
3170    }
3171
3172    /// Retry only persistence after a completed implementation turn.
3173    async fn persist_candidates(&mut self) -> Result<()> {
3174        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3175        for label in labels {
3176            let i = self
3177                .state
3178                .candidates
3179                .iter()
3180                .position(|c| c.label == label)
3181                .context("pending candidate is missing")?;
3182            let c = self.state.candidates[i].clone();
3183            let interrupted = self.state.interrupted_implementations.contains(&label);
3184            let result = async {
3185                let wait = crate::repo_lock::WaitPolicy::new(
3186                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3187                    || false,
3188                );
3189                let _coordination =
3190                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3191                // Best effort evidence requires no index writes. A successful snapshot
3192                // never authorizes removal of a tree whose rescue failed.
3193                if let Err(e) = git::snapshot_uncommitted(
3194                    &c.worktree,
3195                    &self
3196                        .state
3197                        .dir()
3198                        .join("artifacts")
3199                        .join(format!("candidate-{label}-recovery")),
3200                )
3201                .await
3202                {
3203                    self.state.event(
3204                        "preserve",
3205                        format!("candidate {label} snapshot incomplete: {e:#}"),
3206                    );
3207                }
3208                let rescued = git::rescue_commit(
3209                    &c.worktree,
3210                    &format!("magi: candidate {label} (uncommitted work)"),
3211                )
3212                .await?;
3213                self.state.note_withheld("implement", &rescued.withheld);
3214                let base = &self.state.base_commit;
3215                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3216                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3217                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3218                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3219                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3220                if c.empty && commits > 0 && !patch.trim().is_empty()
3221                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3222                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3223                        || self.state.tally.is_some())
3224                {
3225                    // Keep the old classification and pending marker so every
3226                    // retry stops too. Retain decisions as recovery evidence;
3227                    // only a supported recovery operation may replace them.
3228                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3229                }
3230                let c = &mut self.state.candidates[i];
3231                c.commits = commits;
3232                c.stat = stat;
3233                c.files = files;
3234                let no_changes = commits == 0 || patch.trim().is_empty();
3235                c.empty = no_changes && !interrupted;
3236                if interrupted && no_changes {
3237                    c.failed = None;
3238                    c.verified_noop = None;
3239                }
3240                if !c.empty {
3241                    c.failed = None;
3242                    c.verified_noop = None;
3243                }
3244                self.state.event(
3245                    "preserve",
3246                    format!("candidate {label}: {files} files, {commits} commits"),
3247                );
3248                Ok::<_, anyhow::Error>(())
3249            }
3250            .await;
3251            match result {
3252                Ok(()) => {
3253                    self.state.candidate_preservation.remove(&label);
3254                    if !interrupted || self.state.candidates[i].files > 0 {
3255                        self.state.interrupted_implementations.remove(&label);
3256                    }
3257                }
3258                Err(e) => {
3259                    let reason = format!("candidate {label} preservation failed: {e:#}");
3260                    self.state.event("preserve", &reason);
3261                    self.state.candidate_preservation.insert(label, reason);
3262                }
3263            }
3264            if self.state.candidate_preservation.is_empty()
3265                && self.state.status == RunStatus::Blocked
3266                && self.state.candidates.iter().all(|c| c.failed.is_none())
3267            {
3268                // Persist the recovered phase with the cleared pending marker;
3269                // a crash between saves must not turn this into a selected-seat
3270                // failure hold on the next resume.
3271                self.state.status = RunStatus::Implementing;
3272                self.state.event(
3273                    "preserve",
3274                    "candidate persistence recovered; implementation can continue",
3275                );
3276            }
3277            self.state.save()?;
3278        }
3279        if !self.state.candidate_preservation.is_empty() {
3280            self.state.status = RunStatus::Blocked;
3281            self.state.save()?;
3282            bail!(
3283                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3284            );
3285        }
3286        Ok(())
3287    }
3288
3289    /// Ask again, once, for work a CLI did and then failed to hand over.
3290    ///
3291    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3292    /// status with an empty response and a usage report showing output tokens,
3293    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3294    /// seven minutes and 14,267 output tokens that arrived as an empty
3295    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3296    ///
3297    /// Two conditions, and both matter:
3298    ///
3299    /// - **Only when the tree is untouched.** Often the agent has already
3300    ///   written its files and only the closing message was lost; the rescue
3301    ///   commit below picks that up and there is nothing to ask for. Re-asking
3302    ///   then would pay for a second implementation of work already on disk.
3303    /// - **Once.** A CLI that drops one stream can drop the next, and this
3304    ///   node is the most expensive in the graph.
3305    ///
3306    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3307    /// dropped reply still carried its `conversation_id`, so the seat is asked
3308    /// to finish what it was doing rather than sent the whole task again. It
3309    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3310    /// node's - for the same reason a re-ranked judge does: restating finished
3311    /// work is not the work.
3312    ///
3313    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3314    /// same way until it resets, while an abandoned conversation is still
3315    /// there to be picked up.
3316    async fn resume_undelivered(
3317        &mut self,
3318        results: &mut [(usize, SeatState, AgentOutcome)],
3319        sent: &[SeatJob],
3320        prompts: &Prompts,
3321        run_id: &str,
3322    ) {
3323        for (wi, seat, out) in results.iter_mut() {
3324            let Some(dropped) = (match &*out {
3325                AgentOutcome::Dropped(o) => o.dropped.clone(),
3326                _ => None,
3327            }) else {
3328                continue;
3329            };
3330            let Some(job) = sent.get(*wi) else { continue };
3331            // Already on disk? Then only the closing message was lost.
3332            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3333                self.state.event(
3334                    "implement",
3335                    format!(
3336                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3337                         work is in the tree",
3338                        seat.key, dropped.output_tokens, dropped.why
3339                    ),
3340                );
3341                continue;
3342            }
3343            // The re-ask only makes sense as a resume: `resume_after_drop`
3344            // says nothing about the task, trusting the seat to still hold it.
3345            // Without a session to resume — sessions disabled, or this CLI's
3346            // drop shape happened not to carry a session id — that prompt
3347            // would open a brand-new conversation with no context at all,
3348            // which is worse than leaving this as the ordinary failure it
3349            // already is.
3350            if !has_context(&job.spec, seat, job.sessions) {
3351                self.state.event(
3352                    "implement",
3353                    format!(
3354                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3355                         is no session left to resume",
3356                        seat.key, dropped.output_tokens, dropped.why
3357                    ),
3358                );
3359                continue;
3360            }
3361            self.state.event(
3362                "implement",
3363                format!(
3364                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3365                     conversation",
3366                    seat.key, dropped.output_tokens, dropped.why
3367                ),
3368            );
3369            let mut retry = job.clone();
3370            retry.seat = seat.clone();
3371            retry.prompt = prompt::resume_after_drop(&dropped.why);
3372            retry.timeout = retry_budget(job.timeout, true);
3373            retry.stem = format!("{}-resume", job.stem);
3374            let cache = self.state.config.cache_dir();
3375            let ctx = WaveCtx {
3376                carry_seats: false,
3377                pauses: Some((&self.pause, &self.interrupt)),
3378                run: run_id,
3379                node: "implement",
3380                prompts,
3381                cache: cache.as_deref(),
3382                round: None,
3383            };
3384            let (resumed_seat, resumed) =
3385                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3386            *seat = resumed_seat;
3387            *out = resumed;
3388        }
3389    }
3390
3391    /// Fall an implement seat through to the next untried agent in the
3392    /// implementer roster when it lost to quota — or, since the handover was
3393    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3394    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3395    /// quota path itself being unchanged — instead of leaving the
3396    /// seat's loss final the moment one agent's account runs dry.
3397    ///
3398    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3399    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3400    /// truncates to the single slot rotation picked, so a solo task whose one
3401    /// implementer hits quota mid-run used to have nothing else to try. This
3402    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3403    /// unrotated roster — which is the only place the *other* candidates in
3404    /// the machine's roster still exist once `implementers` has been cut down
3405    /// to size.
3406    ///
3407    /// Walks forward from just past the seat's own original position in the
3408    /// roster, never wrapping back to the front: a later candidate slot (say
3409    /// `beta`, the roster's second entry) must fall through to the *next*
3410    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3411    /// almost certainly a different candidate's own agent already — and once
3412    /// the roster's tail is exhausted there is nothing left to fall through
3413    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3414    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3415    /// let this retry that id forever. The loop keeps falling through until
3416    /// an attempt lands something other than `Quota` or the roster's tail
3417    /// runs out of untried ids, at which point the seat is left exactly as
3418    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3419    /// `QuotaLoss` recorded, the candidate failed/empty.
3420    ///
3421    /// `sent` is taken mutably and updated with the fallback agent's spec:
3422    /// `resume_unconfirmed_commands`, which runs after this and also reads
3423    /// `sent`, must see whichever agent actually ended up answering the seat
3424    /// — reading the stale, original spec there would check session
3425    /// eligibility against the wrong CLI and could hand a fallback agent's
3426    /// session id to the agent that just lost the seat to quota.
3427    ///
3428    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3429    /// — `self.seat` only reuses state when the agent id is unchanged, so
3430    /// handing it a different id already gets this for free. Reusing the old
3431    /// seat would resume a different CLI's session as if it were a
3432    /// continuation of this one.
3433    ///
3434    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3435    /// a quota loss cuts an agent off mid-turn, so anything already in the
3436    /// tree is unfinished work, not a completed candidate a re-ask would pay
3437    /// for twice. A dirty tree is rescued into a commit first (the same
3438    /// neutral-identity rescue `implement`'s own outcome loop gives every
3439    /// candidate) so the next agent starts clean.
3440    ///
3441    /// The new agent gets the implementer's full prompt and full
3442    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3443    /// it has no session and no context, and is implementing the task from
3444    /// nothing, unlike a resumed drop which is only restating work already
3445    /// done.
3446    ///
3447    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3448    /// `implement` event, never into `self.state.quota` — that is what
3449    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3450    /// go unspent, and a seat that ultimately recovered on its second or
3451    /// third agent is not the stalled panel that check exists to catch. Only
3452    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3453    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3454    /// outcome loop already has — this helper never pushes to it itself.
3455    async fn resume_seat_handovers(
3456        &mut self,
3457        results: &mut [(usize, SeatState, AgentOutcome)],
3458        sent: &mut [SeatJob],
3459        prompts: &Prompts,
3460        run_id: &str,
3461    ) -> Result<()> {
3462        let instruction = seeded_instruction(&self.state);
3463        let language = self.state.config.graph.language.clone();
3464        let github_language = self.state.config.graph.github_language.clone();
3465        let brief = self
3466            .state
3467            .advice
3468            .as_ref()
3469            .and_then(|a| a.synthesis.as_deref())
3470            .map(str::to_owned);
3471        let attachments = self.state.attachments.clone();
3472        for (wi, seat, out) in results.iter_mut() {
3473            // Who holds the other candidate seats of this wave right now
3474            // (earlier handovers already written back to `sent`).
3475            let others: BTreeSet<String> = sent
3476                .iter()
3477                .enumerate()
3478                .filter(|(j, _)| j != wi)
3479                .map(|(_, j)| j.spec.id.clone())
3480                .collect();
3481            let Some(job) = sent.get_mut(*wi) else {
3482                continue;
3483            };
3484            // Where the seat's own original agent sits in the roster — the
3485            // fallback walk starts just past here, never at the front, so a
3486            // later candidate slot's quota loss does not fall back onto an
3487            // earlier slot's own agent.
3488            let start = self
3489                .roles
3490                .implementer_roster
3491                .iter()
3492                .position(|s| s.id == job.spec.id)
3493                .unwrap_or(0);
3494            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3495            let mut fallback_attempt = 0usize;
3496            let mut prev: Option<FailClass> = None;
3497            while let Some(cur) = FailClass::of(&*out) {
3498                if !should_hand_over(prev.as_ref(), &cur) {
3499                    break;
3500                }
3501                let Some(next) =
3502                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3503                        .cloned()
3504                else {
3505                    break;
3506                };
3507                tried.insert(next.id.clone());
3508                fallback_attempt += 1;
3509
3510                let coordination = self.coordinate(&job.cwd).await?;
3511                if let Ok(r) = git::rescue_commit(
3512                    &job.cwd,
3513                    &format!(
3514                        "magi: candidate {} (uncommitted work before {} fallback)",
3515                        seat.key,
3516                        if cur == FailClass::Quota {
3517                            "quota"
3518                        } else {
3519                            "handover"
3520                        }
3521                    ),
3522                )
3523                .await
3524                {
3525                    self.state.note_withheld("implement", &r.withheld);
3526                }
3527                drop(coordination);
3528
3529                record_handover(
3530                    &mut self.state,
3531                    "implement",
3532                    &seat.key,
3533                    &seat.agent,
3534                    &next.id,
3535                    &cur,
3536                    &fail_reason(&*out),
3537                );
3538                prev = Some(cur.clone());
3539
3540                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3541                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3542                // Kept in sync on `sent` itself, not just the local retry: a
3543                // later helper (`resume_unconfirmed_commands`) reads `sent`
3544                // after this one returns and must see whichever agent is now
3545                // occupying the seat, not the one that just quota'd out —
3546                // otherwise it would judge session/continuation eligibility
3547                // by the wrong CLI and could resend a fallback's session id
3548                // to the agent that lost it the seat in the first place.
3549                job.spec = next.clone();
3550                let mut retry = job.clone();
3551                retry.seat = new_seat;
3552                retry.prompt = prompt::implement_in(
3553                    &instruction,
3554                    &job.cwd.to_string_lossy(),
3555                    &language,
3556                    &github_language,
3557                    brief.as_deref(),
3558                    &attachments,
3559                );
3560                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3561                let cache = self.state.config.cache_dir();
3562                let ctx = WaveCtx {
3563                    carry_seats: false,
3564                    pauses: Some((&self.pause, &self.interrupt)),
3565                    run: run_id,
3566                    node: "implement",
3567                    prompts,
3568                    cache: cache.as_deref(),
3569                    round: None,
3570                };
3571                let (fallback_seat, fallback_out) = run_one(
3572                    retry,
3573                    Arc::clone(&self.sem),
3574                    &ctx,
3575                    &mut self.state,
3576                    fallback_attempt,
3577                )
3578                .await;
3579                *seat = fallback_seat;
3580                *out = fallback_out;
3581            }
3582        }
3583        Ok(())
3584    }
3585
3586    /// Ask an implement seat's own CLI to confirm what it started, once, when
3587    /// its reply reported a command whose completion status it never
3588    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3589    /// that does and does not mean.
3590    ///
3591    /// The completion contract this task asks for, extended to `implement`
3592    /// with the same signal `continue_fix_report` reads for the fixer,
3593    /// rather than a keyword search over the reply or a hard requirement on
3594    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3595    /// a candidate's CLI turn ended cleanly while a test run it had started
3596    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3597    /// named in it at all is untouched by this: `commands` is empty, so
3598    /// there is nothing to be unconfirmed.
3599    ///
3600    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3601    /// this is not about recovering edits that might already be on disk, it
3602    /// is about a result the seat itself never vouched for, which resuming
3603    /// asks for regardless of what the tree already holds. Bounded to one
3604    /// attempt for the same reason `resume_undelivered` is — this is the
3605    /// most expensive node in the graph — and a seat that still cannot
3606    /// confirm on that attempt is left as whatever its (possibly still
3607    /// unconfirmed) reply says; this does not invent a new "failed" reason
3608    /// for a candidate that otherwise produced a real, committed change.
3609    async fn resume_unconfirmed_commands(
3610        &mut self,
3611        results: &mut [(usize, SeatState, AgentOutcome)],
3612        sent: &[SeatJob],
3613        prompts: &Prompts,
3614        run_id: &str,
3615    ) {
3616        for (wi, seat, out) in results.iter_mut() {
3617            let AgentOutcome::Ok(o) = &*out else {
3618                continue;
3619            };
3620            if !has_unconfirmed_command(&o.commands) {
3621                continue;
3622            }
3623            let Some(job) = sent.get(*wi) else { continue };
3624            if !has_context(&job.spec, seat, job.sessions) {
3625                self.state.event(
3626                    "implement",
3627                    format!(
3628                        "{}: the reply named a command whose own CLI never confirmed the exit \
3629                         status of, but there is no session left to resume",
3630                        seat.key
3631                    ),
3632                );
3633                continue;
3634            }
3635            self.state.event(
3636                "implement",
3637                format!(
3638                    "{}: the reply named a command whose own CLI never confirmed the exit \
3639                     status of; resuming the conversation",
3640                    seat.key
3641                ),
3642            );
3643            let mut retry = job.clone();
3644            retry.seat = seat.clone();
3645            retry.prompt = prompt::resume_incomplete(
3646                "a command in your last reply had no confirmed exit status",
3647            );
3648            retry.timeout = retry_budget(job.timeout, true);
3649            retry.stem = format!("{}-confirm", job.stem);
3650            let cache = self.state.config.cache_dir();
3651            let ctx = WaveCtx {
3652                carry_seats: false,
3653                pauses: Some((&self.pause, &self.interrupt)),
3654                run: run_id,
3655                node: "implement",
3656                prompts,
3657                cache: cache.as_deref(),
3658                round: None,
3659            };
3660            let (resumed_seat, resumed) =
3661                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3662            *seat = resumed_seat;
3663            *out = resumed;
3664        }
3665    }
3666
3667    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3668    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3669    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3670    /// that motivated this.
3671    ///
3672    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3673    /// own nudge loop already covers for judge/review/vote seats, and not a
3674    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3675    /// implement seats: here the CLI turn genuinely finished while the node's
3676    /// own work — the fixer's account of what it did — had not. Gated purely
3677    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3678    /// reply, never on any wording in it, so a fixer whose valid, first-try
3679    /// `FixReport` happens to mention having waited on a background test is
3680    /// never resumed — the `Ok(report)` branch at the call site returns
3681    /// before this is ever invoked.
3682    ///
3683    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3684    /// attempt ([`retry_budget`]), nothing attempted once the session is
3685    /// gone, and a quota hit ends the loop immediately rather than retrying a
3686    /// rate limit that fails the same way again.
3687    async fn continue_fix_report(
3688        &mut self,
3689        mut seat: SeatState,
3690        parse_err: String,
3691        job: &SeatJob,
3692        prompts: &Prompts,
3693        run_id: &str,
3694        round: usize,
3695    ) -> (
3696        SeatState,
3697        Option<FixReport>,
3698        Option<String>,
3699        ContinuationRecord,
3700    ) {
3701        let mut last_err = parse_err;
3702        let mut cumulative_wait_ms = 0u64;
3703        let mut attempts = 0usize;
3704        loop {
3705            if !has_context(&job.spec, &seat, job.sessions) {
3706                self.state.event(
3707                    "fix",
3708                    format!(
3709                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3710                         session left to resume into"
3711                    ),
3712                );
3713                let outcome = if attempts == 0 {
3714                    ContinuationOutcome::NoSession
3715                } else {
3716                    ContinuationOutcome::Exhausted
3717                };
3718                return (
3719                    seat,
3720                    None,
3721                    Some(format!("unparsable fix report: {last_err}")),
3722                    ContinuationRecord {
3723                        attempts,
3724                        cumulative_wait_ms,
3725                        outcome,
3726                    },
3727                );
3728            }
3729            if attempts >= MAX_FIX_CONTINUATIONS {
3730                self.state.event(
3731                    "fix",
3732                    format!(
3733                        "round {round}: fixer's reply still had no adoption report after \
3734                         {attempts} continuation(s) ({last_err}); giving up"
3735                    ),
3736                );
3737                return (
3738                    seat,
3739                    None,
3740                    Some(format!(
3741                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3742                    )),
3743                    ContinuationRecord {
3744                        attempts,
3745                        cumulative_wait_ms,
3746                        outcome: ContinuationOutcome::Exhausted,
3747                    },
3748                );
3749            }
3750            attempts += 1;
3751            self.state.event(
3752                "fix",
3753                format!(
3754                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3755                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3756                ),
3757            );
3758            let mut retry = job.clone();
3759            retry.seat = seat.clone();
3760            retry.prompt = prompt::resume_incomplete(&last_err);
3761            retry.timeout = retry_budget(job.timeout, true);
3762            retry.stem = format!("{}-continue{attempts}", job.stem);
3763            let cache = self.state.config.cache_dir();
3764            let ctx = WaveCtx {
3765                carry_seats: false,
3766                pauses: Some((&self.pause, &self.interrupt)),
3767                run: run_id,
3768                node: "fix",
3769                prompts,
3770                cache: cache.as_deref(),
3771                round: Some(round),
3772            };
3773            let (resumed_seat, resumed_out) = run_one(
3774                retry,
3775                Arc::clone(&self.sem),
3776                &ctx,
3777                &mut self.state,
3778                attempts,
3779            )
3780            .await;
3781            seat = resumed_seat;
3782            match resumed_out {
3783                AgentOutcome::Ok(o) => {
3784                    cumulative_wait_ms += o.duration_ms;
3785                    match verdict::extract_json::<FixReport>(&o.text) {
3786                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3787                            self.state.event(
3788                                "fix",
3789                                format!(
3790                                    "round {round}: fixer's adoption report recovered after \
3791                                     {attempts} continuation(s)"
3792                                ),
3793                            );
3794                            return (
3795                                seat,
3796                                Some(report),
3797                                None,
3798                                ContinuationRecord {
3799                                    attempts,
3800                                    cumulative_wait_ms,
3801                                    outcome: ContinuationOutcome::Resumed,
3802                                },
3803                            );
3804                        }
3805                        // The report parsed, but this same reply's own
3806                        // CommandEvidence — the identical record `state.jobs`
3807                        // renders — names a command whose CLI never
3808                        // confirmed an exit status. Read together, that is
3809                        // not a resolved answer: keep nudging rather than
3810                        // accept a report standing next to a command the
3811                        // seat's own CLI cannot vouch for.
3812                        Ok(_) => {
3813                            last_err = "the reply parsed, but it reported a command whose own CLI \
3814                                 never confirmed an exit status"
3815                                .to_owned();
3816                        }
3817                        Err(e) => last_err = e.to_string(),
3818                    }
3819                }
3820                AgentOutcome::Quota(o) => {
3821                    cumulative_wait_ms += o.duration_ms;
3822                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3823                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3824                    self.state.quota.push(loss);
3825                    self.state.event(
3826                        "fix",
3827                        format!(
3828                            "round {round}: continuation rate limited (quota); not retrying now"
3829                        ),
3830                    );
3831                    return (
3832                        seat,
3833                        None,
3834                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3835                        ContinuationRecord {
3836                            attempts,
3837                            cumulative_wait_ms,
3838                            outcome: ContinuationOutcome::QuotaLost,
3839                        },
3840                    );
3841                }
3842                AgentOutcome::Dropped(o) => {
3843                    cumulative_wait_ms += o.duration_ms;
3844                    let why = o
3845                        .dropped
3846                        .as_ref()
3847                        .map(|d| d.why.as_str())
3848                        .unwrap_or("the CLI ended the stream without delivering its answer");
3849                    last_err = format!("the CLI dropped the stream ({why})");
3850                }
3851                AgentOutcome::Failed(e) => last_err = e,
3852                // The continuation was never started; the report stays lost
3853                // and the caller records that, without any handover.
3854                AgentOutcome::ResourceBlocked(e) => {
3855                    return (
3856                        seat,
3857                        None,
3858                        Some(format!("the continuation was not started: {e}")),
3859                        ContinuationRecord {
3860                            attempts: attempts.saturating_sub(1),
3861                            cumulative_wait_ms,
3862                            outcome: ContinuationOutcome::NoSession,
3863                        },
3864                    );
3865                }
3866            }
3867        }
3868    }
3869
3870    fn after_implement(&mut self) -> Result<()> {
3871        if !self.state.candidate_preservation.is_empty() {
3872            bail!("candidate preservation incomplete; judging is unsafe");
3873        }
3874        // Scan every candidate patch once the set is complete.
3875        if self.state.leaks.is_empty() {
3876            let cfg = self.state.config.blind.clone();
3877            let mut leaks = Vec::new();
3878            for c in &self.state.candidates {
3879                let Some(patch) =
3880                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3881                else {
3882                    continue;
3883                };
3884                leaks.extend(blind::scan(
3885                    &format!("candidate {} patch", c.label),
3886                    &patch,
3887                    &cfg.vendor_tokens,
3888                ));
3889            }
3890            if !leaks.is_empty() {
3891                let summary = leaks
3892                    .iter()
3893                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3894                    .collect::<Vec<_>>()
3895                    .join(", ");
3896                match cfg.on_leak {
3897                    LeakPolicy::Fail => {
3898                        self.state.status = RunStatus::Failed;
3899                        self.state
3900                            .event("blind", format!("vendor text in a patch: {summary}"));
3901                        self.state.leaks = leaks;
3902                        self.state.save()?;
3903                        self.settle_questions();
3904                        bail!(
3905                            "blind.on_leak = \"fail\" and vendor text reached a \
3906                             judged patch: {summary}"
3907                        );
3908                    }
3909                    LeakPolicy::Redact => self.state.event(
3910                        "blind",
3911                        format!("redacting vendor text for judging: {summary}"),
3912                    ),
3913                    LeakPolicy::Warn => self.state.event(
3914                        "blind",
3915                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3916                    ),
3917                }
3918                self.state.leaks = leaks;
3919            }
3920        }
3921
3922        if self.state.viable().is_empty() {
3923            if self.state.all_candidates_verified_noop() {
3924                // Every candidate agreed, with evidence the adoption guard
3925                // accepted, that nothing belongs in this worktree. That is
3926                // not the same fact as a candidate that simply failed to
3927                // write anything, and settling it as an ordinary `Failed`
3928                // (see `SCHEMA`'s doc for schema 10) is what let two of
3929                // task 391f's attempts burn a retry each re-discovering the
3930                // same already-landed fix. Terminal either way, so `judge`
3931                // must never run over an empty candidate set — unlike the
3932                // `Failed` branch below this returns `Ok`, not an error:
3933                // nothing here failed.
3934                self.state.status = RunStatus::VerifiedNoop;
3935                self.state.save()?;
3936                self.settle_questions();
3937                return Ok(());
3938            }
3939            self.state.status = RunStatus::Failed;
3940            self.state.save()?;
3941            self.settle_questions();
3942            bail!("no candidate produced a change; nothing to judge");
3943        }
3944        self.state.status = RunStatus::Judging;
3945        self.state.save()?;
3946        Ok(())
3947    }
3948
3949    // --------------------------------------------------------------- judge
3950
3951    async fn judge(&mut self) -> Result<()> {
3952        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
3953        // agent files with `magi task add` name the run that paid for it. The
3954        // prompt overlay is cloned alongside it because the waves borrow it
3955        // while `self` is mutably borrowed by the node's own bookkeeping.
3956        let run_id = self.state.id.clone();
3957        let prompts = self.state.config.prompts.clone();
3958        if !self.state.judgements.is_empty() || self.state.judge_skipped {
3959            return Ok(());
3960        }
3961        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
3962        if viable.len() == 1 {
3963            // Recorded so this is a one-time event: `judgements` stays empty
3964            // either way, which without this flag is indistinguishable from
3965            // "not yet judged" on the next reentry — and status is left
3966            // untouched, so a later node's conclusion (e.g. `Blocked` after
3967            // the review budget ran out) survives a resume instead of being
3968            // clobbered back to `Judging` by this node running again.
3969            self.state.judge_skipped = true;
3970            self.state.event(
3971                "judge",
3972                format!(
3973                    "only candidate {} produced a change; judging skipped",
3974                    viable[0].label
3975                ),
3976            );
3977            self.state.save()?;
3978            return Ok(());
3979        }
3980        self.state.status = RunStatus::Judging;
3981
3982        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
3983        let language = self.state.config.graph.language.clone();
3984        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
3985        let sessions = self.state.config.graph.sessions;
3986        let artifacts = agent::artifacts_dir(&self.state.dir());
3987        let root = self.state.worktree_root();
3988        let base_short = short(&self.state.base_commit);
3989
3990        let mut jobs = Vec::new();
3991        let mut orders = Vec::new();
3992        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
3993            let order = blind::presentation_order(viable.len(), j, self.state.seed);
3994            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
3995            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
3996            let seat_key = format!("judge-{}", j + 1);
3997            let seat = self.seat(&seat_key, &spec.id);
3998            jobs.push(SeatJob {
3999                prompt: prompt::judge(
4000                    &self.state.instruction,
4001                    &views,
4002                    self.roles.judges.len(),
4003                    &base_short,
4004                    &language,
4005                ),
4006                spec,
4007                seat,
4008                cwd: root.join(format!("judge-{}", j + 1)),
4009                timeout,
4010                allow_write: false,
4011                sessions,
4012                artifacts: artifacts.clone(),
4013                stem: format!("judge-{}", j + 1),
4014                handover: None,
4015            });
4016        }
4017
4018        self.state.event(
4019            "judge",
4020            format!(
4021                "{} judges ranking {} candidates blind",
4022                jobs.len(),
4023                viable.len()
4024            ),
4025        );
4026        let labels_for_check = labels.clone();
4027        let mut quota_losses = Vec::new();
4028        let cache = self.state.config.cache_dir();
4029        let ctx = WaveCtx {
4030            carry_seats: false,
4031            pauses: Some((&self.pause, &self.interrupt)),
4032            run: &run_id,
4033            node: "judge",
4034            prompts: &prompts,
4035            cache: cache.as_deref(),
4036            round: None,
4037        };
4038        let results = ask_json_wave::<Ranking>(
4039            jobs,
4040            Arc::clone(&self.sem),
4041            self.state.config.graph.retries,
4042            &self.roles.judge_roster,
4043            &ctx,
4044            &mut quota_losses,
4045            &mut self.state,
4046            &move |r: &Ranking| r.validate(&labels_for_check),
4047        )
4048        .await;
4049        if self.state.resource_block.is_some() {
4050            return self.stop_for_resource();
4051        }
4052        self.state.quota.extend(quota_losses);
4053
4054        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4055            let agent_id = seat.agent.clone();
4056            self.state.seats.insert(seat.key.clone(), seat);
4057            let mut record = Judgement {
4058                judge: j + 1,
4059                seat: format!("judge-{}", j + 1),
4060                agent: agent_id,
4061                ranking: Vec::new(),
4062                reasons: BTreeMap::new(),
4063                confidence: None,
4064                strengths: BTreeMap::new(),
4065                order: orders[j].clone(),
4066                failed: None,
4067                duration_ms: 0,
4068            };
4069            match res {
4070                Ok((ranking, out)) => {
4071                    record.ranking = ranking.normalized();
4072                    record.reasons = ranking.reasons;
4073                    record.strengths = ranking.strengths;
4074                    record.confidence = ranking.confidence;
4075                    record.duration_ms = out.duration_ms;
4076                    self.state.event(
4077                        "judge",
4078                        format!(
4079                            "judge {} ranked {}",
4080                            j + 1,
4081                            record.ranking.iter().collect::<String>()
4082                        ),
4083                    );
4084                }
4085                Err(e) => {
4086                    record.failed = Some(e.to_string());
4087                    self.state
4088                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4089                }
4090            }
4091            self.state.judgements.push(record);
4092            self.state.save()?;
4093        }
4094        Ok(())
4095    }
4096
4097    // ---------------------------------------------------------- deliberate
4098
4099    async fn deliberate(&mut self) -> Result<()> {
4100        if self.hold_base_sync().await? {
4101            return Ok(());
4102        }
4103        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4104        // agent files with `magi task add` name the run that paid for it. The
4105        // prompt overlay is cloned alongside it because the waves borrow it
4106        // while `self` is mutably borrowed by the node's own bookkeeping.
4107        let run_id = self.state.id.clone();
4108        let prompts = self.state.config.prompts.clone();
4109        let rounds = self.state.config.graph.deliberate_rounds;
4110        // Resume unfinished rounds after an outage, but never reopen a tally.
4111        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4112            return Ok(());
4113        }
4114        let tops: Vec<char> = self
4115            .state
4116            .judgements
4117            .iter()
4118            .filter_map(|j| j.ranking.first().copied())
4119            .collect();
4120        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4121            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4122                self.state.event(
4123                    "deliberate",
4124                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4125                );
4126            }
4127            self.state.status = RunStatus::Voting;
4128            self.state.save()?;
4129            return Ok(());
4130        }
4131
4132        self.state.status = RunStatus::Deliberating;
4133        self.state.event(
4134            "deliberate",
4135            format!(
4136                "split: first choices were {} — opening {rounds} round(s)",
4137                tops.iter().collect::<String>()
4138            ),
4139        );
4140
4141        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4142        let language = self.state.config.graph.language.clone();
4143        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4144        let sessions = self.state.config.graph.sessions;
4145        let artifacts = agent::artifacts_dir(&self.state.dir());
4146        let root = self.state.worktree_root();
4147        let base_short = short(&self.state.base_commit);
4148
4149        // Judges argue in sequence so that a turn can answer the one before it;
4150        // that is the difference between deliberation and three parallel
4151        // monologues.
4152        for round in self.state.deliberation.len() + 1..=rounds {
4153            let mut turns: Vec<DeliberationTurn> = Vec::new();
4154            let mut round_losses = Vec::new();
4155            let mut asked = 0;
4156            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4157                if self.state.judgements[j].failed.is_some() {
4158                    continue;
4159                }
4160                let seat_key = format!("judge-{}", j + 1);
4161                let spec = self.occupant(&seat_key, spec);
4162                let mut seat = self.seat(&seat_key, &spec.id);
4163                let transcript = self.transcript(&turns, j);
4164                let build = |context: Option<&str>| {
4165                    prompt::deliberate(
4166                        &self.state.instruction,
4167                        context,
4168                        &transcript,
4169                        round,
4170                        rounds,
4171                        &language,
4172                    )
4173                };
4174                let block = self.candidate_block(&viable, &base_short);
4175                let full = build(Some(&block));
4176                let text = if has_context(&spec, &seat, sessions) {
4177                    build(None)
4178                } else {
4179                    full.clone()
4180                };
4181                let job = SeatJob {
4182                    spec,
4183                    seat: seat.clone(),
4184                    prompt: text,
4185                    cwd: root.join(format!("judge-{}", j + 1)),
4186                    timeout,
4187                    allow_write: false,
4188                    sessions,
4189                    artifacts: artifacts.clone(),
4190                    stem: format!("delib-{round}-judge-{}", j + 1),
4191                    handover: Some(full),
4192                };
4193                let cache = self.state.config.cache_dir();
4194                let ctx = WaveCtx {
4195                    carry_seats: false,
4196                    pauses: Some((&self.pause, &self.interrupt)),
4197                    run: &run_id,
4198                    node: "deliberate",
4199                    prompts: &prompts,
4200                    cache: cache.as_deref(),
4201                    round: None,
4202                };
4203                // A turn is never nudged (`retries` 0): a failed seat is
4204                // handed to the next roster agent, which gets the full
4205                // context. An empty answer is a turn, not a failure.
4206                let mut losses = Vec::new();
4207                asked += 1;
4208                let mut results = ask_wave_with::<String>(
4209                    vec![job],
4210                    Arc::clone(&self.sem),
4211                    0,
4212                    &self.roles.judge_roster,
4213                    &ctx,
4214                    &mut losses,
4215                    &mut self.state,
4216                    &|text: &str| {
4217                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4218                    },
4219                )
4220                .await;
4221                if self.state.resource_block.is_some() {
4222                    return self.stop_for_resource();
4223                }
4224                round_losses.extend(losses.iter().cloned());
4225                self.state.quota.extend(losses);
4226                let (updated, res, _) = results.pop().expect("one job in, one result out");
4227                seat = updated;
4228                let agent_id = seat.agent.clone();
4229                self.state.seats.insert(seat.key.clone(), seat);
4230                let body = match res {
4231                    Ok((body, _)) => body,
4232                    // Skip the seat; a CLI's raw error JSON is never read as
4233                    // this judge's position.
4234                    Err(e) => {
4235                        self.state
4236                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4237                        continue;
4238                    }
4239                };
4240                let tentative = verdict::extract_json::<Position>(&body)
4241                    .ok()
4242                    .and_then(|p| p.tentative)
4243                    .and_then(|s| s.trim().chars().next())
4244                    .map(|c| c.to_ascii_uppercase());
4245                self.state.event(
4246                    "deliberate",
4247                    format!(
4248                        "round {round}: judge {} now favours {}",
4249                        j + 1,
4250                        tentative.map_or("—".to_owned(), |c| c.to_string())
4251                    ),
4252                );
4253                turns.push(DeliberationTurn {
4254                    judge: j + 1,
4255                    agent: agent_id,
4256                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4257                    tentative,
4258                });
4259            }
4260            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4261                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4262                return self.stop_for_resource();
4263            }
4264            self.state
4265                .deliberation
4266                .push(DeliberationRound { round, turns });
4267            self.state.save()?;
4268        }
4269
4270        self.state.status = RunStatus::Voting;
4271        self.state.save()?;
4272        Ok(())
4273    }
4274
4275    // ---------------------------------------------------------------- vote
4276
4277    async fn vote(&mut self) -> Result<()> {
4278        if self.hold_base_sync().await? {
4279            return Ok(());
4280        }
4281        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4282        // agent files with `magi task add` name the run that paid for it. The
4283        // prompt overlay is cloned alongside it because the waves borrow it
4284        // while `self` is mutably borrowed by the node's own bookkeeping.
4285        let run_id = self.state.id.clone();
4286        let prompts = self.state.config.prompts.clone();
4287        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4288            return Ok(());
4289        }
4290        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4291        if viable.len() == 1 {
4292            return Ok(());
4293        }
4294        self.state.status = RunStatus::Voting;
4295
4296        let language = self.state.config.graph.language.clone();
4297        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4298        let sessions = self.state.config.graph.sessions;
4299        let artifacts = agent::artifacts_dir(&self.state.dir());
4300        let root = self.state.worktree_root();
4301        let base_short = short(&self.state.base_commit);
4302        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4303
4304        let mut jobs = Vec::new();
4305        let mut seats_at = Vec::new();
4306        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4307            if self
4308                .state
4309                .judgements
4310                .get(j)
4311                .is_some_and(|r| r.failed.is_some())
4312            {
4313                continue;
4314            }
4315            let seat_key = format!("judge-{}", j + 1);
4316            let spec = self.occupant(&seat_key, spec);
4317            let seat = self.seat(&seat_key, &spec.id);
4318            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4319            let text = if has_context(&spec, &seat, sessions) {
4320                prompt::final_vote(&viable, &language)
4321            } else {
4322                full.clone()
4323            };
4324            jobs.push(SeatJob {
4325                spec,
4326                seat,
4327                prompt: text,
4328                cwd: root.join(format!("judge-{}", j + 1)),
4329                timeout,
4330                allow_write: false,
4331                sessions,
4332                artifacts: artifacts.clone(),
4333                stem: format!("vote-judge-{}", j + 1),
4334                handover: Some(full),
4335            });
4336            seats_at.push(j);
4337        }
4338
4339        self.state.event(
4340            "vote",
4341            format!(
4342                "collecting {} final votes one by one, privately",
4343                jobs.len()
4344            ),
4345        );
4346        let allowed = viable.clone();
4347        let mut quota_losses = Vec::new();
4348        let cache = self.state.config.cache_dir();
4349        let ctx = WaveCtx {
4350            carry_seats: false,
4351            pauses: Some((&self.pause, &self.interrupt)),
4352            run: &run_id,
4353            node: "vote",
4354            prompts: &prompts,
4355            cache: cache.as_deref(),
4356            round: None,
4357        };
4358        let results = ask_json_wave::<FinalVote>(
4359            jobs,
4360            Arc::clone(&self.sem),
4361            self.state.config.graph.retries,
4362            &self.roles.judge_roster,
4363            &ctx,
4364            &mut quota_losses,
4365            &mut self.state,
4366            &move |v: &FinalVote| match v.label() {
4367                Some(c) if allowed.contains(&c) => Ok(()),
4368                other => bail!("vote {other:?} is not one of {allowed:?}"),
4369            },
4370        )
4371        .await;
4372        if self.state.resource_block.is_some() {
4373            return self.stop_for_resource();
4374        }
4375        self.state.quota.extend(quota_losses);
4376
4377        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4378            let agent_id = seat.agent.clone();
4379            self.state.seats.insert(seat.key.clone(), seat);
4380            let initial = self
4381                .state
4382                .judgements
4383                .get(j)
4384                .and_then(|r| r.ranking.first().copied());
4385            let mut record = VoteRecord {
4386                judge: j + 1,
4387                agent: agent_id,
4388                vote: None,
4389                reason: String::new(),
4390                changed: false,
4391            };
4392            match res {
4393                Ok((v, _)) => {
4394                    record.vote = v.label();
4395                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4396                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4397                    self.state.event(
4398                        "vote",
4399                        format!(
4400                            "judge {} voted {}{}",
4401                            j + 1,
4402                            record.vote.unwrap_or('?'),
4403                            if record.changed { " (changed)" } else { "" }
4404                        ),
4405                    );
4406                }
4407                Err(e) => {
4408                    self.state
4409                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4410                }
4411            }
4412            self.state.votes.push(record);
4413            self.state.save()?;
4414        }
4415        Ok(())
4416    }
4417
4418    // --------------------------------------------------------------- tally
4419
4420    fn tally(&mut self) -> Result<()> {
4421        if self.state.tally.is_some() {
4422            return Ok(());
4423        }
4424        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4425        let tops: Vec<char> = self
4426            .state
4427            .judgements
4428            .iter()
4429            .filter_map(|j| j.ranking.first().copied())
4430            .collect();
4431        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4432
4433        // A judge whose private vote failed still counted once, in the initial
4434        // ranking; using it beats discarding a whole seat.
4435        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4436        let mut cast: Vec<char> = Vec::new();
4437        for (i, j) in self.state.judgements.iter().enumerate() {
4438            let vote = self
4439                .state
4440                .votes
4441                .iter()
4442                .find(|v| v.judge == i + 1)
4443                .and_then(|v| v.vote)
4444                .or_else(|| j.ranking.first().copied());
4445            if let Some(v) = vote {
4446                *first_choice.entry(v).or_insert(0) += 1;
4447                cast.push(v);
4448            }
4449        }
4450
4451        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4452        for j in &self.state.judgements {
4453            let n = j.ranking.len();
4454            for (pos, label) in j.ranking.iter().enumerate() {
4455                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4456            }
4457        }
4458
4459        let best = first_choice.values().copied().max().unwrap_or(0);
4460        let mut leaders: Vec<char> = first_choice
4461            .iter()
4462            .filter(|(_, v)| **v == best)
4463            .map(|(k, _)| *k)
4464            .collect();
4465        let mut tie_break = None;
4466        if leaders.len() > 1 {
4467            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4468            let borda_leaders: Vec<char> = leaders
4469                .iter()
4470                .copied()
4471                .filter(|l| borda[l] == top_borda)
4472                .collect();
4473            tie_break = Some(if borda_leaders.len() == 1 {
4474                format!(
4475                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4476                    leaders.len()
4477                )
4478            } else {
4479                format!(
4480                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4481                    leaders.len()
4482                )
4483            });
4484            leaders = borda_leaders;
4485            leaders.sort_unstable();
4486        }
4487        let winner = *leaders
4488            .first()
4489            .or(viable.first())
4490            .context("no candidate to declare a winner from")?;
4491
4492        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4493        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4494        let deliberated = !self.state.deliberation.is_empty();
4495
4496        // Whose verdict is this? A rate-limited seat is absent even if it
4497        // ranked before the limit hit, so presence is measured against the
4498        // recorded losses, not just "did a ranking ever appear".
4499        let quota_seats: std::collections::BTreeSet<&str> =
4500            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4501        let mut present = 0usize;
4502        for (i, j) in self.state.judgements.iter().enumerate() {
4503            if quota_seats.contains(j.seat.as_str()) {
4504                continue;
4505            }
4506            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4507            let voted = self
4508                .state
4509                .votes
4510                .iter()
4511                .any(|v| v.judge == i + 1 && v.vote.is_some());
4512            if ranked || voted {
4513                present += 1;
4514            }
4515        }
4516        // Strict majority of the configured panel. A bare majority is real
4517        // signal we can act on, while a minority verdict must never stand in
4518        // for a healthy one. A one-candidate run needs no panel at all, and
4519        // `judges` stays `0` rather than the roster size a panel that never
4520        // sat would otherwise be credited with.
4521        let needs_quorum = viable.len() > 1;
4522        let judges_total = if needs_quorum {
4523            self.roles.judges.len()
4524        } else {
4525            0
4526        };
4527        let quorum = if needs_quorum {
4528            judges_total / 2 + 1
4529        } else {
4530            0
4531        };
4532        let met_quorum = !needs_quorum || present >= quorum;
4533        let uncontested = (!needs_quorum).then(|| {
4534            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4535        });
4536
4537        self.state.event(
4538            "tally",
4539            match &uncontested {
4540                Some(reason) => format!("winner {winner} — {reason}"),
4541                None => format!(
4542                    "winner {winner} — votes {} | initial {} | {} changed | \
4543                     {present}/{judges_total} judges{}",
4544                    first_choice
4545                        .iter()
4546                        .map(|(k, v)| format!("{k}:{v}"))
4547                        .collect::<Vec<_>>()
4548                        .join(" "),
4549                    if unanimous_initial {
4550                        "unanimous"
4551                    } else {
4552                        "split"
4553                    },
4554                    changed_votes,
4555                    if met_quorum {
4556                        String::new()
4557                    } else {
4558                        format!(" — below quorum ({quorum} required)")
4559                    },
4560                ),
4561            },
4562        );
4563        if !met_quorum {
4564            self.state.event(
4565                "stall",
4566                format!(
4567                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4568                     the run stops here, resumable"
4569                ),
4570            );
4571        }
4572        self.state.tally = Some(Tally {
4573            first_choice,
4574            borda,
4575            winner,
4576            rankings: tops.len(),
4577            unanimous_initial,
4578            deliberated,
4579            changed_votes,
4580            unanimous_final,
4581            tie_break,
4582            judges: judges_total,
4583            present,
4584            quorum,
4585            met_quorum,
4586            uncontested,
4587        });
4588        self.state.status = if met_quorum {
4589            RunStatus::Reviewing
4590        } else {
4591            RunStatus::Stalled
4592        };
4593        self.state.save()?;
4594        Ok(())
4595    }
4596
4597    // ------------------------------------------------------------- recover
4598
4599    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4600    /// resumed toward completion once the transient cause clears.
4601    ///
4602    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4603    /// it toward the quorum, which is exactly the set of seats whose absence
4604    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4605    /// not depend on which node happened to hit the limit), or an ordinary
4606    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4607    /// seat is never disturbed.
4608    ///
4609    /// A seat that now answers with a usable ranking is "recovered": its
4610    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4611    /// `tally` counts it present again), and its vote re-collected. A seat that
4612    /// still fails keeps its loss and stays absent.
4613    ///
4614    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4615    /// to review/gate/merge), `false` when it is still below quorum (the run
4616    /// stays `Stalled`, still resumable for a later retry).
4617    #[allow(clippy::too_many_lines)]
4618    async fn recover_stall(&mut self) -> Result<bool> {
4619        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4620        // agent files with `magi task add` name the run that paid for it. The
4621        // prompt overlay is cloned alongside it because the waves borrow it
4622        // while `self` is mutably borrowed by the node's own bookkeeping.
4623        let run_id = self.state.id.clone();
4624        let prompts = self.state.config.prompts.clone();
4625        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4626        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4627        // is what keeps a non-quota collapse — or a quota loss recorded at the
4628        // deliberate node — from being a permanent dead-end on `--resume`.
4629        let quota_seats: BTreeSet<&str> =
4630            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4631        let absent: Vec<String> = self
4632            .state
4633            .judgements
4634            .iter()
4635            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4636            .map(|j| j.seat.clone())
4637            .collect();
4638        if absent.is_empty() {
4639            return Ok(false);
4640        }
4641        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4642        if viable.len() <= 1 {
4643            return Ok(false);
4644        }
4645        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4646        let language = self.state.config.graph.language.clone();
4647        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4648        let sessions = self.state.config.graph.sessions;
4649        let artifacts = agent::artifacts_dir(&self.state.dir());
4650        let root = self.state.worktree_root();
4651        let base_short = short(&self.state.base_commit);
4652        let candidates: Vec<Candidate> = viable.clone();
4653
4654        // Map each absent seat key to its 0-based position in `roles.judges`.
4655        let mut positions: Vec<usize> = absent
4656            .iter()
4657            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4658            .collect();
4659        if positions.is_empty() {
4660            return Ok(false);
4661        }
4662        positions.sort_unstable();
4663        positions.dedup();
4664
4665        // Re-rank the lost seats, one blind prompt each.
4666        let mut judge_jobs = Vec::new();
4667        for &j in &positions {
4668            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4669            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4670            let seat_key = format!("judge-{}", j + 1);
4671            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4672            let seat = self.seat(&seat_key, &spec.id);
4673            judge_jobs.push(SeatJob {
4674                spec,
4675                seat,
4676                prompt: prompt::judge(
4677                    &self.state.instruction,
4678                    &views,
4679                    self.roles.judges.len(),
4680                    &base_short,
4681                    &language,
4682                ),
4683                cwd: root.join(seat_key),
4684                timeout,
4685                allow_write: false,
4686                sessions,
4687                artifacts: artifacts.clone(),
4688                stem: format!("judge-{}-recover", j + 1),
4689                handover: None,
4690            });
4691        }
4692
4693        let labels_for_check = labels.clone();
4694        let mut judge_losses = Vec::new();
4695        let retries = self.state.config.graph.retries;
4696        let cache = self.state.config.cache_dir();
4697        let ctx = WaveCtx {
4698            carry_seats: false,
4699            pauses: Some((&self.pause, &self.interrupt)),
4700            run: &run_id,
4701            node: "judge",
4702            prompts: &prompts,
4703            cache: cache.as_deref(),
4704            round: None,
4705        };
4706        let results = ask_json_wave::<Ranking>(
4707            judge_jobs,
4708            Arc::clone(&self.sem),
4709            retries,
4710            &self.roles.judge_roster,
4711            &ctx,
4712            &mut judge_losses,
4713            &mut self.state,
4714            &move |r: &Ranking| r.validate(&labels_for_check),
4715        )
4716        .await;
4717        if self.state.resource_block.is_some() {
4718            self.stop_for_resource()?;
4719            return Ok(false);
4720        }
4721
4722        // Refresh the judgement of every seat that ranked again.
4723        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4724        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4725            let agent_id = seat.agent.clone();
4726            self.state.seats.insert(seat.key.clone(), seat);
4727            let record = &mut self.state.judgements[j];
4728            match res {
4729                Ok((ranking, out)) => {
4730                    record.agent = agent_id;
4731                    record.ranking = ranking.normalized();
4732                    record.reasons = ranking.reasons;
4733                    record.strengths = ranking.strengths;
4734                    record.confidence = ranking.confidence;
4735                    record.failed = None;
4736                    record.duration_ms = out.duration_ms;
4737                    recovered.insert(j);
4738                    self.state.event(
4739                        "recover",
4740                        format!("judge {} ranked again after the limit", j + 1),
4741                    );
4742                }
4743                Err(e) => {
4744                    self.state
4745                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4746                }
4747            }
4748        }
4749
4750        // Re-ask the votes of the seats that recovered a ranking.
4751        let mut vote_jobs = Vec::new();
4752        let mut vote_pos: Vec<usize> = Vec::new();
4753        for &j in &recovered {
4754            let seat_key = format!("judge-{}", j + 1);
4755            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4756            let seat = self.seat(&seat_key, &spec.id);
4757            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4758            let text = if has_context(&spec, &seat, sessions) {
4759                prompt::final_vote(&labels, &language)
4760            } else {
4761                full.clone()
4762            };
4763            vote_jobs.push(SeatJob {
4764                spec,
4765                seat,
4766                prompt: text,
4767                cwd: root.join(seat_key),
4768                timeout,
4769                allow_write: false,
4770                sessions,
4771                artifacts: artifacts.clone(),
4772                stem: format!("vote-judge-{}-recover", j + 1),
4773                handover: Some(full),
4774            });
4775            vote_pos.push(j);
4776        }
4777        let allowed = labels.clone();
4778        let mut vote_losses = Vec::new();
4779        let vote_retries = self.state.config.graph.retries;
4780        let vote_cache = self.state.config.cache_dir();
4781        let ctx = WaveCtx {
4782            carry_seats: false,
4783            pauses: Some((&self.pause, &self.interrupt)),
4784            run: &run_id,
4785            node: "vote",
4786            prompts: &prompts,
4787            cache: vote_cache.as_deref(),
4788            round: None,
4789        };
4790        let votes = ask_json_wave::<FinalVote>(
4791            vote_jobs,
4792            Arc::clone(&self.sem),
4793            vote_retries,
4794            &self.roles.judge_roster,
4795            &ctx,
4796            &mut vote_losses,
4797            &mut self.state,
4798            &move |v: &FinalVote| match v.label() {
4799                Some(c) if allowed.contains(&c) => Ok(()),
4800                other => bail!("vote {other:?} is not one of {allowed:?}"),
4801            },
4802        )
4803        .await;
4804        if self.state.resource_block.is_some() {
4805            self.stop_for_resource()?;
4806            return Ok(false);
4807        }
4808        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4809            let agent_id = seat.agent.clone();
4810            self.state.seats.insert(seat.key.clone(), seat);
4811            match res {
4812                Ok((v, _)) => {
4813                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4814                        rec.vote = v.label();
4815                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4816                    } else {
4817                        self.state.votes.push(VoteRecord {
4818                            judge: j + 1,
4819                            agent: agent_id,
4820                            vote: v.label(),
4821                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4822                            changed: false,
4823                        });
4824                    }
4825                    self.state.event(
4826                        "recover",
4827                        format!("judge {} voted again after the limit", j + 1),
4828                    );
4829                }
4830                Err(e) => {
4831                    self.state
4832                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4833                }
4834            }
4835        }
4836
4837        // A seat that ranked again is present even if its re-vote failed —
4838        // `tally` falls back to the initial ranking's first choice — so clear
4839        // its quota loss. Seats that still fail keep theirs and stay absent.
4840        let recovered_keys: BTreeSet<String> = recovered
4841            .iter()
4842            .map(|&j| format!("judge-{}", j + 1))
4843            .collect();
4844        self.state
4845            .quota
4846            .retain(|q| !recovered_keys.contains(&q.seat));
4847        // A seat that hit the limit again is a fresh loss, not the old one:
4848        // replace the stale entry so the history stays one-per-seat and the
4849        // daemon can tell this attempt's loss from a previous session's.
4850        for loss in judge_losses.into_iter().chain(vote_losses) {
4851            if recovered_keys.contains(&loss.seat) {
4852                continue;
4853            }
4854            self.state.quota.retain(|q| q.seat != loss.seat);
4855            self.state.quota.push(loss);
4856        }
4857
4858        // Recompute the verdict from the refreshed panel.
4859        self.state.tally = None;
4860        self.tally()?;
4861        Ok(self
4862            .state
4863            .tally
4864            .as_ref()
4865            .map(|t| t.met_quorum)
4866            .unwrap_or(false))
4867    }
4868
4869    // ---------------------------------------------------------------- graft
4870
4871    /// Port the strengths the judges attributed to the losing candidates onto
4872    /// the winner, once, between the tally and the review loop.
4873    ///
4874    /// Best-effort by construction: it returns nothing and fails nothing. A
4875    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4876    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4877    /// the run carries on as if the phase were off. See [`graft`] for the
4878    /// decisions and `AGENTS.md` for the invariants.
4879    ///
4880    /// The one exception is a rollback that could not be confirmed: the
4881    /// rejected commit may still be the winner's tip, so the run is blocked
4882    /// (resumable; the record stays `Running`, which makes a resume retry the
4883    /// restore) instead of reviewing, gating and merging it.
4884    async fn graft(&mut self) -> Result<()> {
4885        let Err(e) = self.graft_inner().await else {
4886            return Ok(());
4887        };
4888        if e.is::<crate::repo_lock::ResourceBlocked>() {
4889            return Err(e);
4890        }
4891        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4892            self.state.status = RunStatus::Blocked;
4893            self.state.event("graft", format!("blocked: {e:#}"));
4894            self.state.save()?;
4895            return Err(e);
4896        }
4897        // An error after the `Running` record was saved may have left the
4898        // fixer's work in the tree: the record must not stay unsettled
4899        // (a later resume would reset past the review fixes), so confirm the
4900        // restore now, or block.
4901        if let Some(rec) = self.state.graft.clone()
4902            && !rec.outcome.settled()
4903        {
4904            if let Some(winner) = self.state.winner().cloned() {
4905                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4906                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4907                        return Err(re);
4908                    }
4909                    self.state.status = RunStatus::Blocked;
4910                    self.state.event("graft", format!("blocked: {re:#}"));
4911                    self.state.save()?;
4912                    return Err(re);
4913                }
4914            }
4915            if let Some(g) = self.state.graft.as_mut() {
4916                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4917            }
4918        }
4919        self.state.event("graft", format!("skipped: {e:#}"));
4920        if let Err(e) = self.state.save() {
4921            tracing::warn!("save after graft: {e:#}");
4922        }
4923        Ok(())
4924    }
4925
4926    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4927    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4928    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4929        let _coordination = self.coordinate(wt).await?;
4930        crate::recovery::restore_repair(wt, pre)
4931            .await
4932            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4933    }
4934
4935    /// A skip is a run event, recorded once however often the node is re-entered.
4936    fn graft_skipped(&mut self, why: &str) {
4937        let message = format!("skipped: {why}");
4938        if !self
4939            .state
4940            .events
4941            .iter()
4942            .any(|e| e.node == "graft" && e.message == message)
4943        {
4944            self.state.event("graft", message);
4945        }
4946    }
4947
4948    async fn graft_inner(&mut self) -> Result<()> {
4949        // A round that was started and never finished (crash, kill): the
4950        // fixer may have left anything in the tree, so put the winner back.
4951        if let Some(rec) = self.state.graft.clone()
4952            && !rec.outcome.settled()
4953        {
4954            // Reviews exist only if the graft ended without its record being
4955            // settled (a swallowed error): the tree has moved on since, and
4956            // a reset to `pre` would discard the review fixes.
4957            if self.state.reviews.is_empty()
4958                && let Some(winner) = self.state.winner().cloned()
4959            {
4960                self.graft_restore(&winner.worktree, &rec.pre).await?;
4961            }
4962            if rec.outcome == GraftOutcome::WaitingForRestore {
4963                self.state.graft = None;
4964                self.state.save()?;
4965            } else {
4966                if let Some(g) = self.state.graft.as_mut() {
4967                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
4968                }
4969                self.state.event(
4970                    "graft",
4971                    "rolled back: the previous round was interrupted; the winner is untouched",
4972                );
4973                self.state.save()?;
4974                return Ok(());
4975            }
4976        }
4977        let plan = match graft::plan(&self.state) {
4978            Ok(plan) => plan,
4979            Err(skip) => {
4980                if skip != graft::Skip::AlreadyDone {
4981                    self.graft_skipped(skip.reason());
4982                    self.state.save()?;
4983                }
4984                return Ok(());
4985            }
4986        };
4987        let Some(winner) = self.state.winner().cloned() else {
4988            return Ok(());
4989        };
4990        let wt = winner.worktree.clone();
4991        if !git::is_clean(&wt).await? {
4992            self.graft_skipped("the winner's worktree has uncommitted changes");
4993            self.state.save()?;
4994            return Ok(());
4995        }
4996        let pre = git::rev_parse(&wt, "HEAD").await?;
4997        // Saved before the fixer is called: a stop from here on is repaired
4998        // by the branch above on the next entry.
4999        self.state.graft = Some(GraftRecord {
5000            pre: pre.clone(),
5001            post: None,
5002            lines: None,
5003            limit: None,
5004            agent: None,
5005            outcome: GraftOutcome::Running,
5006            requested: graft::requested(&plan),
5007            adoption: None,
5008            adoption_error: None,
5009        });
5010        self.state.event(
5011            "graft",
5012            format!(
5013                "porting strengths of candidate(s) {} onto {}",
5014                plan.strengths
5015                    .iter()
5016                    .map(|(l, _, _)| l.to_string())
5017                    .collect::<Vec<_>>()
5018                    .join(", "),
5019                plan.winner
5020            ),
5021        );
5022        self.state.save()?;
5023
5024        let fix_prompt = prompt::graft(
5025            &self.state.instruction,
5026            plan.winner,
5027            &plan.strengths,
5028            &self.state.config.graph.language,
5029        );
5030        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5031        let sessions = self.state.config.graph.sessions;
5032        let artifacts = agent::artifacts_dir(&self.state.dir());
5033        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5034        let (_, seat, out) = self
5035            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5036                SeatJob {
5037                    prompt: fix_prompt.clone(),
5038                    spec,
5039                    seat,
5040                    cwd: wt.clone(),
5041                    timeout,
5042                    allow_write: true,
5043                    sessions,
5044                    artifacts: artifacts.clone(),
5045                    stem: "graft".to_owned(),
5046                    handover: None,
5047                }
5048            })
5049            .await;
5050        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5051            // Preserve the need to restore earlier fallback edits without
5052            // confusing an unstarted seat with an interrupted invocation.
5053            self.state.graft.as_mut().expect("graft record").outcome =
5054                GraftOutcome::WaitingForRestore;
5055            return self.stop_for_resource();
5056        }
5057        let mut why: Option<String> = None;
5058        match &out {
5059            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5060                why = Some("the fixer produced nothing usable".to_owned());
5061            }
5062            AgentOutcome::Ok(_) => {}
5063            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5064            AgentOutcome::Quota(o) => {
5065                self.state.quota.push(QuotaLoss::from_report(
5066                    seat.key.clone(),
5067                    "graft",
5068                    o.quota.as_ref(),
5069                ));
5070                why = Some("rate limited (quota)".to_owned());
5071            }
5072            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5073                why = Some(format!(
5074                    "the fixer failed: {}",
5075                    e.lines()
5076                        .next()
5077                        .unwrap_or("")
5078                        .chars()
5079                        .take(160)
5080                        .collect::<String>()
5081                ));
5082            }
5083        }
5084        let agent_id = seat.agent.clone();
5085        self.state.seats.insert(seat.key.clone(), seat);
5086
5087        let mut post = None;
5088        let mut lines = None;
5089        let _coordination = self.coordinate(&wt).await?;
5090        if why.is_none() {
5091            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5092            if let Ok(r) = &rescued {
5093                self.state.note_withheld("graft", &r.withheld);
5094            }
5095            let head = git::rev_parse(&wt, "HEAD").await?;
5096            // What is measured must be what gets reviewed and gated: a failed
5097            // rescue, or anything still uncommitted (withheld lockfiles
5098            // aside, which are never part of the tree), rejects the graft.
5099            let dirty = match &rescued {
5100                Ok(r) => git::status_porcelain(&wt)
5101                    .await
5102                    .map(|st| {
5103                        st.lines().any(|l| {
5104                            !r.withheld
5105                                .iter()
5106                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5107                        })
5108                    })
5109                    .unwrap_or(true),
5110                Err(_) => true,
5111            };
5112            if dirty {
5113                why = Some("the fixer left changes that could not be committed".to_owned());
5114            } else if head == pre {
5115                why = Some("the fixer changed nothing".to_owned());
5116            } else {
5117                lines = git::changed_lines(&wt, &pre, &head).await;
5118                post = Some(head);
5119            }
5120        }
5121        let (adoption, adoption_error) = match &out {
5122            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5123            _ => (None, None),
5124        };
5125        let requested = graft::requested(&plan);
5126        let outcome = match why {
5127            None => {
5128                self.state.event(
5129                    "graft",
5130                    format!(
5131                        "applied: {}",
5132                        lines.map_or_else(
5133                            || "changed lines unavailable".to_owned(),
5134                            |n| format!("{n} changed lines")
5135                        )
5136                    ),
5137                );
5138                GraftOutcome::Applied
5139            }
5140            Some(why) => {
5141                // Not settled until the restore is confirmed: the record
5142                // stays `Running`, so a resume retries it.
5143                crate::recovery::restore_repair(&wt, &pre)
5144                    .await
5145                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5146                self.state.event(
5147                    "graft",
5148                    format!("rolled back: {why}; the winner is untouched"),
5149                );
5150                GraftOutcome::RolledBack(why)
5151            }
5152        };
5153        self.state.graft = Some(GraftRecord {
5154            pre,
5155            post,
5156            lines,
5157            limit: None,
5158            agent: Some(agent_id),
5159            outcome,
5160            requested,
5161            adoption,
5162            adoption_error,
5163        });
5164        self.state.save()?;
5165        Ok(())
5166    }
5167
5168    // ----------------------------------------------------------------- fold
5169
5170    async fn fold_losers(&mut self) -> Result<()> {
5171        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5172            return Ok(());
5173        };
5174        if !self.state.candidate_preservation.is_empty() {
5175            bail!("candidate preservation incomplete; refusing cleanup");
5176        }
5177        let repo = self.state.repo.clone();
5178        let _coordination = self.coordinate(&repo).await?;
5179        let mut folded = Vec::new();
5180        for i in 0..self.state.candidates.len() {
5181            let c = &self.state.candidates[i];
5182            if c.label == winner || c.folded {
5183                continue;
5184            }
5185            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5186            remove_preserved_worktree(&repo, &wt).await?;
5187            if git::branch_exists(&repo, &branch).await?
5188                && !git::branch_delete(&repo, &branch).await?
5189            {
5190                bail!("candidate branch removal refused");
5191            }
5192            self.state.candidates[i].folded = true;
5193            self.state.save()?;
5194            folded.push(label.to_string());
5195        }
5196        // The judges are finished; their checkouts are pure cost from here.
5197        let root = self.state.worktree_root();
5198        for j in 1..=self.roles.judges.len() {
5199            let wt = root.join(format!("judge-{j}"));
5200            if wt.exists() {
5201                remove_preserved_worktree(&repo, &wt).await?;
5202            }
5203        }
5204        // The design-deliberation stage is finished by the time a tally
5205        // exists — same reasoning as the judges above.
5206        if self.state.config.graph.advise {
5207            for k in 1..=self.state.config.graph.advisors {
5208                let wt = root.join(format!("advisor-{k}"));
5209                if wt.exists() {
5210                    remove_preserved_worktree(&repo, &wt).await?;
5211                }
5212            }
5213        }
5214        if !folded.is_empty() {
5215            self.state
5216                .event("fold", format!("folded candidates {}", folded.join(", ")));
5217            self.state.save()?;
5218        }
5219        Ok(())
5220    }
5221
5222    // ------------------------------------------------------------ base sync
5223
5224    async fn hold_base_sync(&mut self) -> Result<bool> {
5225        self.hold_base_sync_as(None).await
5226    }
5227
5228    async fn hold_base_sync_kind(
5229        &mut self,
5230        kind: crate::base_sync_recovery::StopKind,
5231    ) -> Result<()> {
5232        self.hold_base_sync_as(Some(kind)).await?;
5233        Ok(())
5234    }
5235
5236    async fn hold_base_sync_as(
5237        &mut self,
5238        kind: Option<crate::base_sync_recovery::StopKind>,
5239    ) -> Result<bool> {
5240        if !self.state.base_sync_blocked() {
5241            self.state.base_sync_recovery.stop = None;
5242            return Ok(false);
5243        }
5244        let head = match self.state.winner() {
5245            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5246                .await
5247                .unwrap_or_default(),
5248            None => String::new(),
5249        };
5250        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5251            && !authorization.consumed
5252            && authorization.stop.head != head
5253        {
5254            authorization.refuse(format!(
5255                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5256                authorization.stop.head
5257            ));
5258        }
5259        self.state.record_base_sync_stop(&head);
5260        if let Some(kind) = kind {
5261            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5262            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5263                stop.kind = kind;
5264            }
5265        }
5266        self.state.save()?;
5267        Ok(true)
5268    }
5269
5270    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5271    /// anything verifies it.
5272    ///
5273    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5274    /// read whatever is checked out in the winner's worktree. Left alone that
5275    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5276    /// it when the run *branched* - and a run takes long enough that the base
5277    /// has usually moved by the time it gets here. A gate that ran there
5278    /// answers "green on the commit this run started from", not "green on
5279    /// what is about to land", and the difference showed up three times in
5280    /// one day as a green run whose merge would have reverted a file another
5281    /// pull request had already landed.
5282    ///
5283    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5284    /// `land::Step::Rebase` calls, rather than a second implementation of the
5285    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5286    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5287    /// doc warns about ("two answers to a question nobody notices until a
5288    /// diff is wrong").
5289    ///
5290    /// Foreign-checkout reviews instead merge the base into the owned shadow
5291    /// worktree, preserving the ancestry required for fast-forward delivery.
5292    /// A conflicting merge is aborted and held for operator resolution.
5293    ///
5294    /// A conflict is not the end of the road: the standing rebase is handed
5295    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5296    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5297    /// land). Once it finishes, review and the gate run as usual on the
5298    /// rebased tree, which is where a breakage the new base caused is caught
5299    /// by the ordinary gate-fix round. magi resolves nothing itself.
5300    ///
5301    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5302    /// the current durable recovery interval, bounds how often the base is
5303    /// synchronized during active recovery (lifetime attempts remain evidence).
5304    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5305    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5306    /// what was tried (rounds spent, paths still conflicted) and the branch
5307    /// and worktree stay exactly as they were - untouched, for a person to
5308    /// look at - which is also what makes re-entering this function
5309    /// afterwards a no-op instead of a second attempt at the same wall. A
5310    /// push failure ends the same way.
5311    async fn sync_to_base(&mut self) -> Result<()> {
5312        if self.state.status == RunStatus::AlreadyInBase {
5313            return Ok(());
5314        }
5315        self.hold_base_sync().await?;
5316        let conflicted = self
5317            .state
5318            .base_sync
5319            .as_ref()
5320            .is_some_and(|s| s.conflict.is_some());
5321        let Some(winner) = self.state.winner().cloned() else {
5322            return Ok(());
5323        };
5324
5325        let repo = self.state.repo.clone();
5326        let remote = self.state.config.merge.remote.clone();
5327        let base_branch = self.state.base_branch.clone();
5328        let tracking = format!("{remote}/{base_branch}");
5329
5330        let fetched_base =
5331            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5332        // No network, or the remote never had this branch: the run already
5333        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5334        // otherwise), and one that got this far is not blocked by a fetch
5335        // that fails now. It just has no newer tip to compare against.
5336        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5337            return Ok(());
5338        };
5339
5340        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5341        self.state.ensure_base_episode();
5342        let mut pending = self
5343            .state
5344            .base_sync_recovery
5345            .operations
5346            .last()
5347            .filter(|o| !o.finished)
5348            .cloned();
5349        let scratch = self.state.dir().join("base-sync");
5350        if git::rebase_in_progress(&scratch).await {
5351            // Older resource stops predate operation reservations. Import the
5352            // standing Git operation, never retarget it to today's main.
5353            if pending.is_none()
5354                && self.state.base_sync_recovery.operations.is_empty()
5355                && !conflicted
5356                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5357                    .await
5358                    .ok()
5359                    .as_deref()
5360                    == Some(head.as_str())
5361                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5362                && git::is_clean(&winner.worktree).await?
5363            {
5364                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5365                if self.state.base_sync.is_none() {
5366                    self.state.base_sync = Some(BaseSync {
5367                        tip: saved_tip.clone(),
5368                        behind: 0,
5369                        attempts: 0,
5370                        conflict: None,
5371                        already_in: None,
5372                    });
5373                }
5374                self.state.reserve_base_operation(&head, &saved_tip);
5375                self.state.event(
5376                    "base-recovery",
5377                    "imported standing legacy rebase as a spent synchronization reservation",
5378                );
5379                self.state.save()?;
5380                pending = self.state.base_sync_recovery.operations.last().cloned();
5381            }
5382            let expected = pending.as_ref().map(|op| op.tip.as_str());
5383            if expected.is_none()
5384                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5385            {
5386                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5387                if self.state.base_sync.is_none() {
5388                    self.state.base_sync = Some(BaseSync {
5389                        tip: tip.clone(),
5390                        behind: 0,
5391                        attempts: 0,
5392                        conflict: None,
5393                        already_in: None,
5394                    });
5395                }
5396                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5397                self.state.event("land", why);
5398                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5399                    .await?;
5400                return Ok(());
5401            }
5402        }
5403        if let Some(op) = &pending {
5404            // A replay may add the pinned base's commits, never foreign work.
5405            let mut replay_proven =
5406                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5407            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5408                let wait = repository_wait(&self.state, None);
5409                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5410                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5411                    Ok(true) => {
5412                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5413                        self.state.save()?;
5414                        git::worktree_remove(&repo, &scratch).await.ok();
5415                        replay_proven = true;
5416                    }
5417                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5418                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5419                }
5420            }
5421            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5422                if self.state.foreign_checkout.is_some() {
5423                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5424                        .await
5425                        .ok()
5426                        .as_deref()
5427                        == Some(op.head.as_str())
5428                        && git::rev_parse(&repo, &format!("{head}^2"))
5429                            .await
5430                            .ok()
5431                            .as_deref()
5432                            == Some(op.tip.as_str());
5433                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5434                    .await
5435                    .is_ok_and(|m| m.is_empty())
5436                    && let Ok(added) =
5437                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5438                {
5439                    replay_proven = true;
5440                    for commit in added {
5441                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5442                            replay_proven = false;
5443                            break;
5444                        }
5445                    }
5446                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5447                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5448                }
5449            }
5450            // Never retarget an operation across a crash, even if main moved.
5451            // sync_to_head resets and cleans with -x, so even ignored untracked
5452            // files must stop recovery independently of the old tracked tree.
5453            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5454                || (head != op.head
5455                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5456                        .await?
5457                        .is_empty())
5458                || (!git::is_clean(&winner.worktree).await?
5459                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5460                        .await
5461                        .is_ok_and(|o| o.ok()))
5462                || !replay_proven
5463            {
5464                let why = format!(
5465                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5466                    op.id
5467                );
5468                let sync = self
5469                    .state
5470                    .base_sync
5471                    .as_mut()
5472                    .context("reserved synchronization has no lifetime accounting")?;
5473                sync.conflict = Some(why.clone());
5474                self.state.event("land", why);
5475                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5476                    .await?;
5477                return Ok(());
5478            }
5479            if head != op.head {
5480                git::sync_to_head(&winner.worktree).await?;
5481            }
5482            tip = op.tip.clone();
5483        }
5484        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5485        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5486        if pending.is_none()
5487            && !conflicted
5488            && fetched_base
5489            && self
5490                .state
5491                .base_sync_recovery
5492                .episodes
5493                .last()
5494                .is_some_and(|e| e.boundary.is_some())
5495            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5496            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5497            && owner.runs.last() == Some(&self.state.id)
5498            && owner.status != crate::queue::TaskStatus::Done
5499            && !owner.fresh_start
5500            && owner.review_branch.is_none()
5501            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5502            && let Some(approval) = owner.resumption.as_ref()
5503            && approval.instruction == owner.instruction
5504            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5505            && git::is_clean(&winner.worktree).await?
5506            && self.state.renew_base_episode(&task, approval, &head, &tip)
5507        {
5508            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5509            self.state.save()?;
5510        }
5511
5512        // Before any rebase, and before a recorded conflict is honoured: a
5513        // branch whose change reached the base under other commit ids has
5514        // nothing to rebase and nothing to conflict with, and a run that
5515        // already stopped on that phantom conflict recovers here on resume.
5516        // `behind == 0` with head == tip is a branch the base has since taken
5517        // in whole, whether or not a conflict was ever recorded: the ancestry
5518        // proof must still run (`classify` ignores a head still on the start
5519        // commit).
5520        if (behind > 0 || conflicted || head == tip)
5521            && self
5522                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5523                .await?
5524        {
5525            return Ok(());
5526        }
5527        if conflicted && self.state.base_recovery_pending() {
5528            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5529            let owner =
5530                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5531            let owner_valid = owner.as_ref().is_ok_and(|t| {
5532                t.runs.last() == Some(&self.state.id)
5533                    && t.status != crate::queue::TaskStatus::Done
5534                    && !t.fresh_start
5535                    && t.review_branch.is_none()
5536            });
5537            let invalidation = if !owner_valid {
5538                Some("latest task ownership changed")
5539            } else if !owner.as_ref().is_ok_and(|t| {
5540                authorization
5541                    .generation
5542                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5543            }) {
5544                Some("owner approval generation changed")
5545            } else if let Some(scope) = &authorization.scope {
5546                if scope.tip != tip {
5547                    Some("fetched base changed")
5548                } else if !self
5549                    .state
5550                    .base_sync_recovery
5551                    .episodes
5552                    .last()
5553                    .is_some_and(|e| e.id == scope.episode)
5554                {
5555                    Some("active synchronization interval changed")
5556                } else if !owner
5557                    .as_ref()
5558                    .is_ok_and(|t| t.instruction == scope.instruction)
5559                {
5560                    Some("task instruction changed")
5561                } else {
5562                    None
5563                }
5564            } else {
5565                None
5566            };
5567            if let Some(reason) = invalidation {
5568                let authorization = self
5569                    .state
5570                    .base_sync_recovery
5571                    .authorizations
5572                    .last_mut()
5573                    .unwrap();
5574                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5575                self.state.save()?;
5576                return Ok(());
5577            }
5578            if !fetched_base
5579                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5580                || !git::is_clean(&winner.worktree).await?
5581                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5582            {
5583                return Ok(());
5584            }
5585        }
5586        let recovery = conflicted && self.state.consume_base_recovery(&head);
5587        if conflicted {
5588            // Consumption (including a refused changed HEAD) is durable before Git.
5589            self.state.save()?;
5590            if !recovery {
5591                return Ok(());
5592            }
5593        }
5594
5595        if behind == 0 {
5596            // A fixer-finished rebase moves the branch ref before the
5597            // winner's worktree is told (`sync_to_head` below). A run that
5598            // died in between resumes here with `behind == 0` and a tree still
5599            // holding the pre-rebase files, which review and the gate would
5600            // then read. That state is exactly: HEAD moved off the tip the
5601            // rebase started from, yet the tree is still identical to that
5602            // tip. A tree with edits of its own differs from it, so nothing
5603            // is thrown away.
5604            if let Some(from) = self
5605                .state
5606                .rebase_fixes
5607                .iter()
5608                .rev()
5609                .find_map(|r| r.from.clone())
5610                && from != head
5611                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5612                    .await
5613                    .is_ok_and(|o| o.ok())
5614            {
5615                git::sync_to_head(&winner.worktree).await?;
5616            }
5617            // An earlier attempt may have rebased the branch locally and died
5618            // before pushing it (only the fresh-rebase arm below pushes).
5619            // Publish it now, so the plain push at PR time is not refused as
5620            // a non-fast-forward. A run already holding a recorded conflict
5621            // never reaches here; that case is out of scope.
5622            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5623            if let Some(why) = &conflict {
5624                self.state.status = RunStatus::Blocked;
5625                self.state.event("land", why.clone());
5626            }
5627            self.state.base_sync = Some(BaseSync {
5628                tip,
5629                behind: 0,
5630                attempts,
5631                conflict,
5632                already_in: None,
5633            });
5634            if pending.is_some() {
5635                self.state.finish_base_operation(Some(&head));
5636            }
5637            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5638                .await?;
5639            self.state.save()?;
5640            return Ok(());
5641        }
5642
5643        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5644            let why = format!(
5645                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5646                 synchronization(s); synchronizing again would only race it",
5647                winner.branch
5648            );
5649            self.state.status = RunStatus::Blocked;
5650            self.state.base_sync = Some(BaseSync {
5651                tip,
5652                behind,
5653                attempts,
5654                conflict: Some(why.clone()),
5655                already_in: None,
5656            });
5657            self.state.event("land", why);
5658            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5659                .await?;
5660            self.state.save()?;
5661            return Ok(());
5662        }
5663
5664        if self.state.foreign_checkout.is_some() {
5665            // Delivery to the operator's checkout is fast-forward only. A
5666            // rebase here would discard its saved head from our ancestry.
5667            let wait = repository_wait(&self.state, None);
5668            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5669            if self.state.base_sync.is_none() {
5670                self.state.base_sync = Some(BaseSync {
5671                    tip: tip.clone(),
5672                    behind,
5673                    attempts: 0,
5674                    conflict: None,
5675                    already_in: None,
5676                });
5677            }
5678            self.state.reserve_base_operation(&head, &tip);
5679            self.state.save()?;
5680            let merged = git::git_raw(
5681                &winner.worktree,
5682                &[
5683                    "-c",
5684                    "core.hooksPath=/dev/null",
5685                    "merge",
5686                    "--no-ff",
5687                    "--no-autostash",
5688                    "--no-edit",
5689                    "-m",
5690                    &format!("Merge {tracking} before verifying"),
5691                    &tip,
5692                ],
5693            )
5694            .await?;
5695            let conflict = if merged.ok() {
5696                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5697                self.state.event(
5698                    "land",
5699                    format!("merged {tracking} into {} before verifying", winner.branch),
5700                );
5701                None
5702            } else {
5703                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5704                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5705                }
5706                let why = format!(
5707                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5708                    winner.branch,
5709                    merged.stderr.chars().take(600).collect::<String>()
5710                );
5711                self.state.status = RunStatus::Blocked;
5712                self.state.event("land", &why);
5713                Some(why)
5714            };
5715            let synced = conflict.is_none();
5716            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5717            self.state
5718                .finish_base_operation(synced.then_some(fresh.as_str()));
5719            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5720            self.state.base_sync = Some(BaseSync {
5721                tip,
5722                behind: if synced { 0 } else { behind },
5723                attempts,
5724                conflict,
5725                already_in: None,
5726            });
5727            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5728                .await?;
5729            self.state.save()?;
5730            return Ok(());
5731        }
5732
5733        self.state.event(
5734            "land",
5735            format!(
5736                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5737                winner.branch
5738            ),
5739        );
5740        self.state.save()?;
5741
5742        // The remote's copy of the branch, read now and only if the fetch
5743        // really succeeded (a stale tracking ref must never pin a lease). It is
5744        // pushed over after a rebase only when it is a commit this branch
5745        // already contains, by ancestry or by patch (an earlier rebase of ours
5746        // that never reached the remote): anything else is somebody else's work.
5747        let branch_tracking = format!("{remote}/{}", winner.branch);
5748        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5749        let remote_tip = if self.state.task_pr_branch().is_none()
5750            && matches!(&fetched_branch, Ok(o) if o.ok())
5751        {
5752            git::rev_parse(&repo, &branch_tracking).await.ok()
5753        } else {
5754            None
5755        };
5756        // A remote tip this branch does not contain is somebody else's work:
5757        // rebasing would leave a local tip that can never be pushed. Stop
5758        // before touching anything and say so.
5759        if let Some(theirs) = &remote_tip
5760            && !git::is_ancestor(&repo, theirs, &head).await
5761            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5762                .await
5763                .is_ok_and(|missing| missing.is_empty())
5764        {
5765            let why = format!(
5766                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5767                 them",
5768                short(theirs),
5769                winner.branch
5770            );
5771            self.state.status = RunStatus::Blocked;
5772            self.state.base_sync = Some(BaseSync {
5773                tip,
5774                behind,
5775                attempts,
5776                conflict: Some(why.clone()),
5777                already_in: None,
5778            });
5779            self.state.event("land", why);
5780            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5781                .await?;
5782            self.state.save()?;
5783            return Ok(());
5784        }
5785
5786        if self.state.base_sync.is_none() {
5787            self.state.base_sync = Some(BaseSync {
5788                tip: tip.clone(),
5789                behind,
5790                attempts: 0,
5791                conflict: None,
5792                already_in: None,
5793            });
5794        }
5795        self.state.reserve_base_operation(&head, &tip);
5796        self.state.save()?;
5797        let scratch = self.state.dir().join("base-sync");
5798        let rebased =
5799            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5800                .await
5801            {
5802                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5803                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5804                Err(e) => Err(e),
5805            };
5806        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5807        let stop_kind = match &rebased {
5808            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5809            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5810            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5811        };
5812        match rebased {
5813            Ok(None) => {
5814                // The branch ref moved, but a worktree that already had it
5815                // checked out (the winner's) was not told; sync its index and
5816                // files before anything reads them.
5817                git::sync_to_head(&winner.worktree).await?;
5818                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5819                self.state.finish_base_operation(Some(&fresh));
5820                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5821                let mut conflict = None;
5822                if let Some(pinned) = &remote_tip {
5823                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5824                    match pushed {
5825                        Ok(o) if o.ok() => self.state.event(
5826                            "land",
5827                            format!("pushed rebased {} to {remote}", winner.branch),
5828                        ),
5829                        Ok(o) => {
5830                            conflict = Some(format!(
5831                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5832                                winner.branch,
5833                                short(pinned),
5834                                o.stderr.chars().take(600).collect::<String>()
5835                            ));
5836                        }
5837                        Err(e) => {
5838                            conflict = Some(format!(
5839                                "rebased {} locally but could not push it: {e:#}",
5840                                winner.branch
5841                            ));
5842                        }
5843                    }
5844                }
5845                if let Some(why) = &conflict {
5846                    self.state.status = RunStatus::Blocked;
5847                    self.state.event("land", why.clone());
5848                }
5849                self.state.base_sync = Some(BaseSync {
5850                    tip: tip.clone(),
5851                    behind: 0,
5852                    attempts,
5853                    conflict,
5854                    already_in: None,
5855                });
5856                self.state
5857                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5858            }
5859            Ok(Some(conflict)) => {
5860                self.state.finish_base_operation(None);
5861                let why = format!(
5862                    "{} conflicts with {tracking} and did not rebase: {}",
5863                    winner.branch,
5864                    conflict.chars().take(600).collect::<String>()
5865                );
5866                self.state.status = RunStatus::Blocked;
5867                self.state.base_sync = Some(BaseSync {
5868                    tip,
5869                    behind,
5870                    attempts,
5871                    conflict: Some(why.clone()),
5872                    already_in: None,
5873                });
5874                self.state.event("land", why);
5875            }
5876            // A refused Git write or a busy repository: nothing is wrong with
5877            // the branch. The scratch rebase stays standing and the run
5878            // resumes into it (`base_sync` must not claim a conflict).
5879            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5880                self.state.save()?;
5881                return Ok(());
5882            }
5883            Err(e) => {
5884                self.state.finish_base_operation(None);
5885                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5886                self.state.status = RunStatus::Blocked;
5887                self.state.base_sync = Some(BaseSync {
5888                    tip,
5889                    behind,
5890                    attempts,
5891                    conflict: Some(why.clone()),
5892                    already_in: None,
5893                });
5894                self.state.event("land", why);
5895            }
5896        }
5897        if !self.state.base_sync_blocked() {
5898            self.state.base_sync_recovery.stop = None;
5899        }
5900        self.hold_base_sync_kind(stop_kind).await?;
5901        self.state.save()?;
5902        Ok(())
5903    }
5904
5905    /// Push a branch an earlier attempt rebased locally but never published,
5906    /// pinned to the remote tip read right after a successful fetch. Returns
5907    /// the reason when the run must stop; `None` when there was nothing to do
5908    /// (no remote copy, the same tip, or a remote copy this branch already
5909    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5910    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5911        // Task competitions publish only after the PR text gate, to their
5912        // public head; never resurrect a candidate branch on the remote.
5913        if self.state.task_pr_branch().is_some() {
5914            return None;
5915        }
5916        let repo = self.state.repo.clone();
5917        let remote = self.state.config.merge.remote.clone();
5918        let fetched = git::fetch(&repo, &remote, branch).await;
5919        if !matches!(&fetched, Ok(o) if o.ok()) {
5920            return None;
5921        }
5922        let branch_tracking = format!("{remote}/{branch}");
5923        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5924        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5925            return None;
5926        }
5927        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5928            .await
5929            .is_ok_and(|missing| missing.is_empty())
5930        {
5931            return Some(format!(
5932                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5933                 them",
5934                short(&theirs)
5935            ));
5936        }
5937        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5938            Ok(o) if o.ok() => {
5939                self.state
5940                    .event("land", format!("pushed rebased {branch} to {remote}"));
5941                None
5942            }
5943            Ok(o) => Some(format!(
5944                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
5945                 someone may have pushed): {}",
5946                short(&theirs),
5947                o.stderr.chars().take(600).collect::<String>()
5948            )),
5949            Err(e) => Some(format!(
5950                "{branch} is rebased locally but could not be pushed: {e:#}"
5951            )),
5952        }
5953    }
5954
5955    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
5956    /// is already on `tip` under other commit ids ([`crate::already`]); returns
5957    /// whether it did.
5958    ///
5959    /// Checked only when the base is ahead of the branch. A failing check is
5960    /// "not proven" - the ordinary rebase path then decides - never a reason to
5961    /// stop the run.
5962    ///
5963    /// The remote copy of the branch is held to the same standard as the local
5964    /// one: if it carries a tip this worktree does not, that tip must itself be
5965    /// proven in the base, or nothing is settled (a pull request would
5966    /// otherwise be closed over commits nobody checked). The pull request is
5967    /// closed *before* the terminal status is saved; if that fails for a
5968    /// reason other than a refusal (no network, a `gh` error) the run is left
5969    /// `Blocked` with the reason as its conflict, which a resume retries -
5970    /// the same recovery a phantom conflict gets.
5971    async fn settle_already_in(
5972        &mut self,
5973        branch: &str,
5974        tip: &str,
5975        head: &str,
5976        attempts: usize,
5977        behind: usize,
5978    ) -> Result<bool> {
5979        let repo = self.state.repo.clone();
5980        let remote = self.state.config.merge.remote.clone();
5981        let start = self.state.base_commit.clone();
5982        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
5983            Ok(Some(e)) => e,
5984            Ok(None) => return Ok(false),
5985            Err(e) => {
5986                tracing::warn!("already-in-base check for {branch}: {e:#}");
5987                return Ok(false);
5988            }
5989        };
5990        let mut verified = vec![head.to_owned()];
5991        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
5992        let fetched = git::fetch(&repo, &remote, &public_branch).await;
5993        if matches!(&fetched, Ok(o) if o.ok())
5994            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
5995            && theirs != head
5996        {
5997            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
5998                Ok(Some(_)) => verified.push(theirs),
5999                _ => return Ok(false),
6000            }
6001        }
6002        let base_branch = self.state.base_branch.clone();
6003        let message = format!(
6004            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6005             land",
6006            evidence.names(),
6007            evidence.proof.as_str()
6008        );
6009        let closed =
6010            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6011                .await;
6012        match closed {
6013            Ok(Ok(url)) => self
6014                .state
6015                .event("land", format!("closed {url}: superseded on {base_branch}")),
6016            Ok(Err(why)) => self
6017                .state
6018                .event("land", format!("did not close a pull request: {why}")),
6019            Err(e) => {
6020                let why = format!(
6021                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6022                     not be closed ({e:#}); resume to retry"
6023                );
6024                self.state.status = RunStatus::Blocked;
6025                self.state.base_sync = Some(BaseSync {
6026                    tip: tip.to_owned(),
6027                    behind,
6028                    attempts,
6029                    conflict: Some(why.clone()),
6030                    already_in: None,
6031                });
6032                self.state.event("land", why);
6033                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6034                    .await?;
6035                self.state.save()?;
6036                return Ok(true);
6037            }
6038        }
6039        self.state.status = RunStatus::AlreadyInBase;
6040        self.state.base_sync_recovery.stop = None;
6041        self.state.base_sync = Some(BaseSync {
6042            tip: tip.to_owned(),
6043            behind,
6044            attempts,
6045            conflict: None,
6046            already_in: Some(evidence),
6047        });
6048        self.state.event("land", message);
6049        self.state.save()?;
6050        self.settle_questions();
6051        Ok(true)
6052    }
6053
6054    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6055    /// last landed the winner on, once it has run, else the commit the run
6056    /// branched from.
6057    ///
6058    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6059    /// and `vote` all happen before there is a winner to rebase, so they
6060    /// compare every candidate against the branch point on purpose, and a
6061    /// base that moves after they are already done cannot change an answer
6062    /// they already gave.
6063    fn landing_base(&self) -> String {
6064        self.state
6065            .base_sync
6066            .as_ref()
6067            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6068    }
6069
6070    // ------------------------------------------------------- operator fix
6071
6072    /// Route specific, already-recorded review findings to a fixer for a
6073    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6074    /// entry point.
6075    ///
6076    /// Distinct from `review_loop`'s own fix step in three ways: it never
6077    /// runs a reviewer wave, it never spends review-round budget, and what
6078    /// happened is recorded as an [`OperatorFixRequest`] appended to
6079    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6080    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6081    /// and vote must never be rewritten to look like a manufactured blocking
6082    /// verdict.
6083    ///
6084    /// Only meaningful once review has actually concluded: `Ready` (handed
6085    /// off with findings still open, or simply concluded clean while minor
6086    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6087    /// gate failed). Everything else is refused: a run still in progress
6088    /// should simply be resumed, and a `Merged` run's branch has already
6089    /// landed — reopening *this* run's own record cannot change that, so the
6090    /// answer there is a fresh `magi review <branch>`.
6091    ///
6092    /// A real commit here re-verifies through a fresh, ordinary review-only
6093    /// run on the same branch ([`Self::review`]) rather than reopening this
6094    /// run's own `review_loop`: once any round in this run's history went
6095    /// clean, `review_conclusion` treats that as permanent by design (the
6096    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6097    /// way to force one more genuine reviewer wave out of *this* run without
6098    /// either rewriting history or weakening that guarantee for every other
6099    /// caller. A review-only run costs nothing extra — no implementation, no
6100    /// judging, no vote — and exercises the exact same review → verify →
6101    /// gate → (human) merge path, unmodified.
6102    pub async fn fix_selected(
6103        &mut self,
6104        ids: &[String],
6105        reason: &str,
6106        allow_stale: bool,
6107    ) -> Result<()> {
6108        let reason = reason.trim();
6109        if reason.is_empty() {
6110            bail!("a fix request needs a reason — that is the operator's own record of why");
6111        }
6112        if ids.is_empty() {
6113            bail!("no finding id given");
6114        }
6115        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6116            bail!(
6117                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6118                 has already concluded — can be given a targeted fix. A run still \
6119                 in progress should simply be resumed; a `merged` run's branch has \
6120                 already landed, so its answer is a fresh `magi review <branch>`, \
6121                 not reopening this run's own record",
6122                self.state.id,
6123                self.state.status.as_str()
6124            );
6125        }
6126        let Some(winner) = self.state.winner().cloned() else {
6127            bail!("run {} has no winning candidate to fix", self.state.id);
6128        };
6129        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6130            bail!(
6131                "branch `{}` no longer exists; this run cannot be extended",
6132                winner.branch
6133            );
6134        }
6135        let home = crate::run::home();
6136        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6137            bail!(
6138                "run {} is currently being worked on by another magi process",
6139                self.state.id
6140            );
6141        }
6142        // Held for the rest of this call, including the follow-up review
6143        // below: two `magi fix` invocations against the same run must not
6144        // both reach the worktree manipulation further down, which would
6145        // otherwise race to remove and recreate the same directory — see
6146        // [`FixClaim`]'s own doc.
6147        let _claim = FixClaim::acquire(&self.state.dir())?;
6148
6149        // Resolve every id before spending anything — an unknown id refuses
6150        // the whole request rather than silently dropping it — and dedup
6151        // while keeping the operator's own order.
6152        let mut seen = BTreeSet::new();
6153        let mut findings = Vec::new();
6154        let mut missing = Vec::new();
6155        for id in ids {
6156            if !seen.insert(id.clone()) {
6157                continue;
6158            }
6159            match self.state.finding(id) {
6160                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6161                    id: f.id.clone(),
6162                    severity: f.severity,
6163                    reviewer_vote: rec.vote,
6164                    round: round.round,
6165                    round_head: round.head.clone(),
6166                    reviewer: rec.reviewer,
6167                    agent: rec.agent.clone(),
6168                    file: f.file.clone(),
6169                    line: f.line,
6170                    title: f.title.clone(),
6171                    detail: f.detail.clone(),
6172                    outcome: OperatorFixOutcome::Pending,
6173                }),
6174                None => missing.push(id.clone()),
6175            }
6176        }
6177        if !missing.is_empty() {
6178            bail!(
6179                "unknown finding id(s): {}; nothing was changed",
6180                missing.join(", ")
6181            );
6182        }
6183
6184        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6185        let stale_details: Vec<(String, String)> = findings
6186            .iter()
6187            .filter(|f| f.round_head != head_at_request)
6188            .map(|f| (f.id.clone(), f.round_head.clone()))
6189            .collect();
6190        let stale = !stale_details.is_empty();
6191        if stale && !allow_stale {
6192            bail!(
6193                "the branch has moved since some finding(s) were raised — {} — now \
6194                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6195                stale_details
6196                    .iter()
6197                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6198                    .collect::<Vec<_>>()
6199                    .join(", "),
6200                short(&head_at_request)
6201            );
6202        }
6203
6204        let request = OperatorFixRequest {
6205            requested_at: Timestamp::now(),
6206            reason: reason.to_owned(),
6207            findings,
6208            head_at_request: head_at_request.clone(),
6209            allow_stale,
6210            stale,
6211            fix: None,
6212            result_head: None,
6213            follow_up_review_run: None,
6214        };
6215        self.state.event(
6216            "fix",
6217            format!(
6218                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6219                request.findings.len(),
6220                request
6221                    .findings
6222                    .iter()
6223                    .map(|f| f.id.as_str())
6224                    .collect::<Vec<_>>()
6225                    .join(", "),
6226            ),
6227        );
6228        // Recorded now, before any worktree work or the fixer call itself —
6229        // and re-saved at each checkpoint below: a crash at any point after
6230        // this (mid fixer call, mid follow-up review) must not lose the fact
6231        // that this was requested, for which findings, and why. Everything
6232        // past this point reads and writes through `request_index` rather
6233        // than a local variable, since `request` itself is moved here.
6234        self.state.operator_fixes.push(request);
6235        self.state.save()?;
6236        let request_index = self.state.operator_fixes.len() - 1;
6237
6238        // A fresh, dedicated worktree for this one call, never the winner's
6239        // own worktree in place: that one may already be gone (folded away),
6240        // and reusing it in place would leave the branch checked out there
6241        // when the follow-up review below tries to check it out again. Freed
6242        // immediately after, either way — but only once confirmed clean:
6243        // `worktree_remove` is a `git worktree remove --force`, which would
6244        // otherwise discard uncommitted work left there by the operator or
6245        // another process before this had a chance to even look at it.
6246        let repo = self.state.repo.clone();
6247        let coordination = self.coordinate(&repo).await?;
6248        if winner.worktree.exists() {
6249            // Lockfiles a rescue commit withheld stay untracked on purpose and
6250            // are already recorded; they are not the operator's work to protect.
6251            let dirty = git::git(
6252                &winner.worktree,
6253                &["status", "--porcelain", "--untracked-files=all"],
6254            )
6255            .await?;
6256            let only_withheld = dirty.lines().all(|l| {
6257                l.strip_prefix("?? ")
6258                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6259            });
6260            if !only_withheld {
6261                bail!(
6262                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6263                     discard them first",
6264                    winner.worktree.display()
6265                );
6266            }
6267            git::worktree_remove(&self.state.repo, &winner.worktree)
6268                .await
6269                .ok();
6270        }
6271        let fix_worktree = self.state.worktree_root().join("operator-fix");
6272        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6273        git::git(
6274            &self.state.repo,
6275            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6276        )
6277        .await
6278        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6279        if !git::is_clean(&fix_worktree).await? {
6280            git::worktree_remove(&self.state.repo, &fix_worktree)
6281                .await
6282                .ok();
6283            bail!(
6284                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6285                winner.branch
6286            );
6287        }
6288        let repo_for_setup = self.state.repo.clone();
6289        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6290            Ok(()) => {
6291                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6292            }
6293            Err(e) => Err(e),
6294        };
6295        if let Err(e) = setup {
6296            git::worktree_remove(&self.state.repo, &fix_worktree)
6297                .await
6298                .ok();
6299            return Err(e);
6300        }
6301
6302        // This explicit operator request retries availability independently
6303        // of execute(); the previous call has already saved and cleaned up.
6304        self.state.resource_block = None;
6305
6306        let run_id = self.state.id.clone();
6307        let prompts = self.state.config.prompts.clone();
6308        let language = self.state.config.graph.language.clone();
6309        let sessions = self.state.config.graph.sessions;
6310        let artifacts = agent::artifacts_dir(&self.state.dir());
6311        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6312            .findings
6313            .iter()
6314            .map(|f| Finding {
6315                id: f.id.clone(),
6316                severity: f.severity,
6317                file: f.file.clone(),
6318                line: f.line,
6319                title: f.title.clone(),
6320                detail: f.detail.clone(),
6321            })
6322            .collect();
6323        let fix_prompt = prompt::operator_fix_in(
6324            &self.state.instruction,
6325            &finding_list,
6326            reason,
6327            &stale_details,
6328            &head_at_request,
6329            &language,
6330            &self.state.config.graph.github_language,
6331        );
6332        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6333        drop(coordination);
6334        let (job, seat, out) = self
6335            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6336                prompt: fix_prompt.clone(),
6337                spec,
6338                seat,
6339                cwd: fix_worktree.clone(),
6340                timeout,
6341                allow_write: true,
6342                sessions,
6343                artifacts: artifacts.clone(),
6344                stem: "operator-fix".to_owned(),
6345                handover: None,
6346            })
6347            .await;
6348        let agent_id = seat.agent.clone();
6349
6350        let mut fix = FixRecord {
6351            agent: agent_id,
6352            addressed: Vec::new(),
6353            rejected: Vec::new(),
6354            notes: String::new(),
6355            committed: false,
6356            failed: None,
6357            duration_ms: 0,
6358            continuation: None,
6359        };
6360        let mut final_seat = seat.clone();
6361        match out {
6362            AgentOutcome::Ok(o) => {
6363                fix.duration_ms = o.duration_ms;
6364                let parsed = verdict::extract_json::<FixReport>(&o.text);
6365                let incomplete_reason = match &parsed {
6366                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6367                        "the reply parsed, but it reported a command whose own CLI \
6368                         never confirmed an exit status"
6369                            .to_owned(),
6370                    ),
6371                    Ok(_) => None,
6372                    Err(e) => Some(e.to_string()),
6373                };
6374                match incomplete_reason {
6375                    None => {
6376                        let report = parsed.expect("checked Ok above");
6377                        fix.addressed = report.addressed;
6378                        fix.rejected = report.rejected;
6379                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6380                    }
6381                    Some(reason) => {
6382                        let (resumed_seat, resolved, failure, cont) = self
6383                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6384                            .await;
6385                        fix.duration_ms += cont.cumulative_wait_ms;
6386                        fix.continuation = Some(cont);
6387                        final_seat = resumed_seat;
6388                        match resolved {
6389                            Some(report) => {
6390                                fix.addressed = report.addressed;
6391                                fix.rejected = report.rejected;
6392                                fix.notes =
6393                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6394                            }
6395                            None => fix.failed = failure,
6396                        }
6397                    }
6398                }
6399            }
6400            AgentOutcome::Dropped(o) => {
6401                fix.duration_ms = o.duration_ms;
6402                let why = o
6403                    .dropped
6404                    .as_ref()
6405                    .map(|d| d.why.as_str())
6406                    .unwrap_or("the CLI ended the stream without delivering its answer");
6407                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6408            }
6409            AgentOutcome::Quota(o) => {
6410                self.state.quota.push(QuotaLoss::from_report(
6411                    final_seat.key.clone(),
6412                    "fix",
6413                    o.quota.as_ref(),
6414                ));
6415                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6416            }
6417            AgentOutcome::Failed(e) => fix.failed = Some(e),
6418            AgentOutcome::ResourceBlocked(e) => {
6419                fix.failed = Some(format!("the fixer was not started: {e}"));
6420            }
6421        }
6422        // Even a deferred call must rescue edits, release the worktree and
6423        // finalize its record below so an explicit `magi fix` can retry.
6424        if fix.continuation.is_none() {
6425            fix.continuation = Some(ContinuationRecord::not_needed());
6426        }
6427        self.state.seats.insert(final_seat.key.clone(), final_seat);
6428
6429        let rescue_message = format!(
6430            "magi: operator-selected fix ({}) (uncommitted work)",
6431            self.state.operator_fixes[request_index]
6432                .findings
6433                .iter()
6434                .map(|f| f.id.as_str())
6435                .collect::<Vec<_>>()
6436                .join(", ")
6437        );
6438        let repo = self.state.repo.clone();
6439        let coordination = self.coordinate(&repo).await?;
6440        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6441            self.state.note_withheld("fix", &r.withheld);
6442        }
6443        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6444        fix.committed = after != head_at_request;
6445        git::worktree_remove(&self.state.repo, &fix_worktree)
6446            .await
6447            .ok();
6448        drop(coordination);
6449
6450        self.state.event(
6451            "fix",
6452            match &fix.failed {
6453                Some(reason) => format!(
6454                    "operator fix: adoption report was lost ({reason}); {}",
6455                    if fix.committed {
6456                        "committed"
6457                    } else {
6458                        "NO new commit"
6459                    }
6460                ),
6461                None => format!(
6462                    "operator fix: {} addressed, {} rejected, {}",
6463                    fix.addressed.len(),
6464                    fix.rejected.len(),
6465                    if fix.committed {
6466                        "committed"
6467                    } else {
6468                        "NO new commit"
6469                    }
6470                ),
6471            },
6472        );
6473
6474        // Every selected finding gets an outcome — never left `Pending` once
6475        // the fixer's own turn is over. A report that never came back at all
6476        // marks every one of them `Unreported`, not silently "not addressed":
6477        // quota, a dropped stream, or an exhausted continuation are gaps in
6478        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6479        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6480        for f in &mut self.state.operator_fixes[request_index].findings {
6481            f.outcome = if fix.failed.is_some() {
6482                OperatorFixOutcome::Unreported
6483            } else if fix.addressed.contains(&f.id) {
6484                OperatorFixOutcome::Addressed
6485            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6486                OperatorFixOutcome::Rejected { why: r.why.clone() }
6487            } else {
6488                OperatorFixOutcome::Unreported
6489            };
6490        }
6491
6492        let committed = fix.committed;
6493        if committed {
6494            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6495        }
6496        self.state.operator_fixes[request_index].fix = Some(fix);
6497        // Saved again now that the fixer's own outcome is final, on top of
6498        // the save right after the request was first pushed above.
6499        self.state.save()?;
6500        if self.state.resource_block.is_some() {
6501            return self.stop_for_resource();
6502        }
6503
6504        if committed {
6505            self.state.event(
6506                "fix",
6507                format!(
6508                    "operator fix committed {}; opening a follow-up review-only run",
6509                    short(&after)
6510                ),
6511            );
6512            // The operator asked for the fix, and the follow-up serves whatever
6513            // task the run it follows served.
6514            let origin =
6515                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6516            match Self::review(
6517                &self.state.repo,
6518                &winner.branch,
6519                self.state.config.clone(),
6520                origin,
6521            )
6522            .await
6523            {
6524                Ok(mut follow_up) => {
6525                    follow_up.state.event(
6526                        "start",
6527                        format!(
6528                            "requested by an operator fix on run {} for finding(s) {}",
6529                            self.state.id,
6530                            self.state.operator_fixes[request_index]
6531                                .findings
6532                                .iter()
6533                                .map(|f| f.id.as_str())
6534                                .collect::<Vec<_>>()
6535                                .join(", "),
6536                        ),
6537                    );
6538                    follow_up.state.save()?;
6539                    let follow_up_id = follow_up.state.id.clone();
6540                    // The follow-up is a run like any other: it belongs to the
6541                    // task of the run it follows, or to one filed for it.
6542                    let adopted = match crate::direct::adopt(
6543                        &follow_up.state,
6544                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6545                    ) {
6546                        Ok(a) => a,
6547                        Err(e) => {
6548                            // An ownerless run must not spend agent calls; it
6549                            // stays saved, and `magi run --resume` adopts it.
6550                            self.state.event(
6551                                "fix",
6552                                format!(
6553                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6554                                ),
6555                            );
6556                            self.state.save()?;
6557                            return Ok(());
6558                        }
6559                    };
6560                    let executed = follow_up.execute().await;
6561                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6562                    if let Some(a) = adopted {
6563                        a.finish(&follow_up.state, executed);
6564                    }
6565                    if let Some(e) = failure {
6566                        self.state.event(
6567                            "fix",
6568                            format!(
6569                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6570                            ),
6571                        );
6572                    }
6573                    self.state.operator_fixes[request_index].follow_up_review_run =
6574                        Some(follow_up_id);
6575                }
6576                Err(e) => {
6577                    self.state.event(
6578                        "fix",
6579                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6580                    );
6581                }
6582            }
6583            self.state.save()?;
6584        }
6585
6586        Ok(())
6587    }
6588
6589    // --------------------------------------------------------------- review
6590
6591    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6592    /// names, in order, each at most once, moving on only when the call
6593    /// advances (an error, a quota hit or nothing usable - the same decision
6594    /// point `agent::chain_advances` is for the other chained roles).
6595    ///
6596    /// Each agent gets its own seat from `Runner::seat`: the same agent
6597    /// continues its conversation (the winner's own implementer seat when it
6598    /// is the winner's author, now that the competition is over), another
6599    /// takes a fresh one so the full prompt is sent again. A handover is
6600    /// recorded under `node`, which is what makes the fallback stick for the
6601    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6602    /// is returned, so an exhausted chain reads exactly like a single failed
6603    /// fixer: one quota loss, the same wording, the same refund. An earlier
6604    /// attempt's edits are left in the tree and judged, with the final
6605    /// attempt's, by what git says afterwards.
6606    ///
6607    /// Returns the job that produced the outcome, for `continue_fix_report`.
6608    async fn ask_fixer(
6609        &mut self,
6610        winner: &Candidate,
6611        node: &'static str,
6612        round: Option<usize>,
6613        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6614    ) -> (SeatJob, SeatState, AgentOutcome) {
6615        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6616        self.ask_fixer_among(attempts, node, round, None, build)
6617            .await
6618    }
6619
6620    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6621    /// worktree and a commit: before the next agent is asked after one that
6622    /// did not satisfy the call, the worktree is put back on that commit, so
6623    /// a failed agent's partial edits are not carried into the next one.
6624    async fn ask_fixer_among(
6625        &mut self,
6626        attempts: Vec<(AgentSpec, String)>,
6627        node: &'static str,
6628        round: Option<usize>,
6629        restore: Option<(&Path, &str)>,
6630        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6631    ) -> (SeatJob, SeatState, AgentOutcome) {
6632        let run_id = self.state.id.clone();
6633        let prompts = self.state.config.prompts.clone();
6634        let cache = self.state.config.cache_dir();
6635        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6636        let mut last = None;
6637        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6638            let seat = self.seat(&key, &spec.id);
6639            let mut job = build(spec.clone(), seat);
6640            if i > 0 {
6641                job.stem = format!("{}-{}", job.stem, spec.id);
6642            }
6643            let ctx = WaveCtx {
6644                carry_seats: false,
6645                pauses: Some((&self.pause, &self.interrupt)),
6646                run: &run_id,
6647                node,
6648                prompts: &prompts,
6649                cache: cache.as_deref(),
6650                round,
6651            };
6652            let (seat, out) =
6653                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6654            // A resource block never advances the chain: the chosen agent
6655            // was not asked, so it must be the one asked again on resume.
6656            let advances = match &out {
6657                AgentOutcome::Ok(o) => agent::output_advances(o),
6658                AgentOutcome::ResourceBlocked(_) => false,
6659                _ => true,
6660            };
6661            if let Some(next) = ids.get(i + 1)
6662                && advances
6663            {
6664                self.state.seats.insert(seat.key.clone(), seat.clone());
6665                let class =
6666                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6667                let reason = match &out {
6668                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6669                    other => fail_reason(other),
6670                };
6671                // A failed agent's edits must not reach the next one, and a
6672                // restore that did not work ends the chain: the call fails
6673                // and the caller's own (confirmed) restore decides.
6674                if let Some((wt, rev)) = restore
6675                    && let Err(e) = self.graft_restore(wt, rev).await
6676                {
6677                    let why =
6678                        format!("could not restore the worktree before the next fixer: {e:#}");
6679                    tracing::warn!("{why}");
6680                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6681                        AgentOutcome::ResourceBlocked(why)
6682                    } else {
6683                        AgentOutcome::Failed(why)
6684                    };
6685                    last = Some((job, seat, out));
6686                    break;
6687                }
6688                record_handover(
6689                    &mut self.state,
6690                    node,
6691                    &seat.key,
6692                    &spec.id,
6693                    next,
6694                    &class,
6695                    &reason,
6696                );
6697                continue;
6698            }
6699            last = Some((job, seat, out));
6700            break;
6701        }
6702        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6703        if let AgentOutcome::Quota(output) = &out
6704            && node != "graft"
6705        {
6706            // Graft is best effort: its caller records the loss and restores
6707            // the pre-graft tree, then proceeds to review and gate.
6708            // Every fallback was tried. Machine outages cannot be repaired by
6709            // spending a fix round; use each caller's resumable resource path.
6710            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6711            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6712            self.state.quota.push(loss);
6713            self.state.seats.insert(seat.key.clone(), seat.clone());
6714            self.state.status = RunStatus::Blocked;
6715            return (
6716                job,
6717                seat,
6718                AgentOutcome::ResourceBlocked(
6719                    "fixer unavailable; restore agent availability and resume".into(),
6720                ),
6721            );
6722        }
6723        (job, seat, out)
6724    }
6725
6726    /// The one repair a verification-only failure earns (see
6727    /// [`crate::recovery`]), before the first review round of a run that took
6728    /// a branch over.
6729    ///
6730    /// Saved as started *before* the fixer is called, so a resume never
6731    /// repairs twice; a start found on re-entry is rolled back to the commit
6732    /// it began from. A failed or unusable fixer leaves the tree as it was and
6733    /// the review goes on over it, so the first round's own verification
6734    /// reports the failure again. The failure is never carried into a round as
6735    /// a result: verification is re-run on whatever commit this leaves.
6736    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6737        use crate::recovery::{Mode, Repair};
6738        if !self.state.reviews.is_empty() {
6739            return Ok(());
6740        }
6741        let Some(mut rec) = self.state.recovery.clone() else {
6742            return Ok(());
6743        };
6744        if rec.mode != Mode::VerifyRepair {
6745            return Ok(());
6746        }
6747        let wt = winner.worktree.clone();
6748        let mut head = git::rev_parse(&wt, "HEAD").await?;
6749        match rec.repair.clone() {
6750            Repair::Done { .. } => return Ok(()),
6751            Repair::WaitingForRestore { before } => {
6752                self.graft_restore(&wt, &before).await?;
6753                head = before;
6754                rec.repair = Repair::NotStarted;
6755                self.state.recovery = Some(rec.clone());
6756                self.state.save()?;
6757            }
6758            Repair::Started { before } => {
6759                // Interrupted: the unfinished edits are not trusted.
6760                // An unconfirmed rollback blocks the run with `Started` kept,
6761                // so a resume retries it; the tree is never reviewed as is.
6762                if let Err(e) = self.graft_restore(&wt, &before).await {
6763                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6764                        return Err(e);
6765                    }
6766                    return self
6767                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6768                        .await;
6769                }
6770                rec.repair = Repair::Done {
6771                    before: before.clone(),
6772                    after: before,
6773                    committed: false,
6774                    failed: Some("interrupted; rolled back".to_owned()),
6775                };
6776                self.state
6777                    .event("recovery", "the repair was interrupted and rolled back");
6778                self.state.recovery = Some(rec);
6779                self.state.save()?;
6780                return Ok(());
6781            }
6782            Repair::NotStarted => {}
6783        }
6784        if crate::recovery::recheck(&mut rec, &head) {
6785            self.state.event("recovery", rec.summary());
6786            self.state.recovery = Some(rec);
6787            self.state.save()?;
6788            return Ok(());
6789        }
6790        rec.repair = Repair::Started {
6791            before: head.clone(),
6792        };
6793        self.state.recovery = Some(rec.clone());
6794        self.state.save()?;
6795
6796        let language = self.state.config.graph.language.clone();
6797        let sessions = self.state.config.graph.sessions;
6798        let artifacts = agent::artifacts_dir(&self.state.dir());
6799        let fix_prompt = prompt::recovery_fix(
6800            &self.state.instruction,
6801            &rec,
6802            self.state.config.graph.review_rounds,
6803            &language,
6804        );
6805        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6806        self.state.event(
6807            "recovery",
6808            format!(
6809                "repair: the fixer starts from the failed verification on {}",
6810                short(&head)
6811            ),
6812        );
6813        let (_job, seat, out) = self
6814            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6815                prompt: fix_prompt.clone(),
6816                spec,
6817                seat,
6818                cwd: wt.clone(),
6819                timeout,
6820                allow_write: true,
6821                sessions,
6822                artifacts: artifacts.clone(),
6823                stem: "fix-recovery".to_owned(),
6824                handover: None,
6825            })
6826            .await;
6827        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6828            rec.repair = Repair::WaitingForRestore { before: head };
6829            self.state.recovery = Some(rec);
6830            return self.stop_for_resource();
6831        }
6832        let failed = match &out {
6833            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6834            AgentOutcome::Quota(o) => {
6835                self.state.quota.push(QuotaLoss::from_report(
6836                    seat.key.clone(),
6837                    "fix",
6838                    o.quota.as_ref(),
6839                ));
6840                Some("rate limited (quota); the fixer could not run".to_owned())
6841            }
6842            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6843            AgentOutcome::Failed(e) => Some(e.clone()),
6844        };
6845        self.state.seats.insert(seat.key.clone(), seat);
6846        let coordination = self.coordinate(&wt).await?;
6847        let failed = match failed {
6848            // The agent's partial edits are not kept.
6849            Some(why) => {
6850                if let Err(e) = git::restore_to(&wt, &head).await {
6851                    return self
6852                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6853                        .await;
6854                }
6855                Some(why)
6856            }
6857            None => {
6858                if let Ok(r) = git::rescue_commit(
6859                    &wt,
6860                    "magi: recovery repair of the failed verification (uncommitted work)",
6861                )
6862                .await
6863                {
6864                    self.state.note_withheld("fix", &r.withheld);
6865                }
6866                None
6867            }
6868        };
6869        let after = git::rev_parse(&wt, "HEAD").await?;
6870        drop(coordination);
6871        self.state.event(
6872            "recovery",
6873            match &failed {
6874                Some(why) => format!("repair: nothing kept ({why})"),
6875                None if after != head => {
6876                    format!("repair: committed {}", short(&after))
6877                }
6878                None => "repair: the fixer made no commit".to_owned(),
6879            },
6880        );
6881        rec.repair = Repair::Done {
6882            before: head.clone(),
6883            committed: after != head,
6884            after,
6885            failed,
6886        };
6887        self.state.recovery = Some(rec);
6888        self.state.save()?;
6889        Ok(())
6890    }
6891
6892    /// A repair whose rollback could not be confirmed: the run is blocked and
6893    /// resumable, and the unconfirmed state is never reviewed.
6894    async fn stop_recovery(
6895        &mut self,
6896        mut rec: crate::recovery::RecoveryRecord,
6897        head: &str,
6898        why: String,
6899    ) -> Result<()> {
6900        // `Started` stays recorded: a resume retries the restore.
6901        rec.repair = crate::recovery::Repair::Started {
6902            before: head.to_owned(),
6903        };
6904        self.state.recovery = Some(rec);
6905        self.state
6906            .event("recovery", format!("repair could not be undone: {why}"));
6907        self.state.status = RunStatus::Blocked;
6908        self.state.save()?;
6909        bail!("recovery repair could not be rolled back: {why}")
6910    }
6911
6912    async fn review_loop(&mut self) -> Result<()> {
6913        if self.hold_base_sync().await? {
6914            return Ok(());
6915        }
6916        loop {
6917            if self
6918                .state
6919                .base_sync
6920                .as_ref()
6921                .is_some_and(|s| s.conflict.is_some())
6922            {
6923                return Ok(());
6924            }
6925            if self.catchup_repairable() {
6926                let record = self.state.reviews.last().expect("repairable round").clone();
6927                let winner = self
6928                    .state
6929                    .winner()
6930                    .cloned()
6931                    .context("catch-up repair has no winner")?;
6932                if !self.fix_review_round(&winner, record).await? {
6933                    return Ok(());
6934                }
6935            }
6936            self.review_loop_pass().await?;
6937            if !self.catchup_repairable() {
6938                return Ok(());
6939            }
6940        }
6941    }
6942
6943    /// Derive an owed catch-up repair from durable round evidence, including
6944    /// reentry after a crash before the fixer was dispatched.
6945    fn catchup_repairable(&self) -> bool {
6946        self.state.reviews.len() < self.state.config.graph.review_rounds
6947            && self.state.reviews.last().is_some_and(|r| {
6948                !r.clean
6949                    && r.blocking == 0
6950                    && r.fix.is_none()
6951                    && r.e2e_status() == E2eStatus::Failed
6952            })
6953    }
6954
6955    async fn review_loop_pass(&mut self) -> Result<()> {
6956        // A base that would not rebase is a person's decision, not a review
6957        // round: nothing here would change the answer, and reviewers and a
6958        // fixer would be spending real budget on a tree that cannot land
6959        // regardless of what they find.
6960        if self
6961            .state
6962            .base_sync
6963            .as_ref()
6964            .is_some_and(|s| s.conflict.is_some())
6965        {
6966            return Ok(());
6967        }
6968        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
6969        // agent files with `magi task add` name the run that paid for it. The
6970        // prompt overlay is cloned alongside it because the waves borrow it
6971        // while `self` is mutably borrowed by the node's own bookkeeping.
6972        let run_id = self.state.id.clone();
6973        let prompts = self.state.config.prompts.clone();
6974        let Some(winner) = self.state.winner().cloned() else {
6975            return Ok(());
6976        };
6977        let granted_review = self
6978            .state
6979            .repair_grants
6980            .last()
6981            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
6982            .cloned();
6983        let fresh_review = granted_review
6984            .as_ref()
6985            .is_some_and(|g| self.state.reviews.len() == g.review_start);
6986        let max_rounds = granted_review
6987            .as_ref()
6988            .map_or(self.state.config.graph.review_rounds, |g| {
6989                g.review_start.saturating_add(1)
6990            });
6991        // A clean round, an exhausted round budget, or a stalled tree (see
6992        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
6993        // they are recorded — recomputed here, not read off `status`, so a
6994        // reentry into a run that already stopped restates the identical
6995        // verdict instead of silently handing back whatever an earlier node
6996        // in this same walk clobbered `status` to (a solo-candidate
6997        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
6998        // below runs an empty range once the budget is spent, and would
6999        // otherwise fall through without touching `status` at all.
7000        if let Some(status) =
7001            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7002        {
7003            // A reentry after a crash between the last round's save and
7004            // `stop_reviewing` reaches the hand-off here, not there.
7005            if status == RunStatus::Gating {
7006                self.record_contested_handoff();
7007            }
7008            self.state.status = status;
7009            self.state.save()?;
7010            return Ok(());
7011        }
7012        self.state.status = RunStatus::Reviewing;
7013        // A last recorded round whose own verification never resolved
7014        // (`ResourceBlocked` — the shared build cache, not the patch) is
7015        // never a concluded round, whatever the round budget says: starting
7016        // a fresh round on top of it would spend a whole new reviewer wave
7017        // re-reading an unchanged patch instead of just retrying the one
7018        // check that actually needs it, and once the budget is spent the
7019        // loop below has nothing left to do at all (its range is empty).
7020        // Retry that check directly instead, exactly the same retry
7021        // `stop_reviewing` already does for its own catch-up case.
7022        if !fresh_review
7023            && self.state.reviews.last().is_some_and(|r| {
7024                r.e2e_status() == E2eStatus::ResourceBlocked
7025                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7026            })
7027        {
7028            let shell = self.state.config.shell();
7029            return self
7030                .stop_unresolved_review(
7031                    "the last round's own verification never resolved",
7032                    &shell,
7033                    &winner.worktree,
7034                )
7035                .await;
7036        }
7037
7038        if granted_review.is_none() {
7039            self.recovery_repair(&winner).await?;
7040        }
7041        if self.resource_halt()? {
7042            return Ok(());
7043        }
7044
7045        let repo = self.state.repo.clone();
7046        let root = self.state.worktree_root();
7047        let language = self.state.config.graph.language.clone();
7048        let sessions = self.state.config.graph.sessions;
7049        let artifacts = self.merge_repair_artifacts();
7050        let base = self.landing_base();
7051        let base_short = short(&base);
7052        let reviewers = self.roles.reviewers.clone();
7053        let shell = self.state.config.shell();
7054
7055        for round in (self.state.reviews.len() + 1)..=max_rounds {
7056            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7057            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7058            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7059            // The prior round's own record, already persisted — never a
7060            // hand-carried variable of just its failing output: that is
7061            // exactly what let a round's e2e result drift out of sync with
7062            // which commit it was actually about (see `SCHEMA`'s doc for
7063            // schema 8). Judged against `head`, the commit reviewers are
7064            // about to look at now, so the summary always reads as "an
7065            // earlier head" here — this round's own patch has not been
7066            // checked yet.
7067            let prev_verification = self
7068                .state
7069                .reviews
7070                .last()
7071                .and_then(|r| r.verification_summary(&head));
7072            // Only the first round of a run that took a branch over: later
7073            // rounds have this run's own history.
7074            let prior_section = match (&self.state.recovery, round) {
7075                (Some(rec), 1) if self.state.reviews.is_empty() => {
7076                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7077                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7078                            Some(crate::recovery::head_of(
7079                                &git::diff(&winner.worktree, reviewed, "HEAD")
7080                                    .await
7081                                    .unwrap_or_default(),
7082                                crate::recovery::SINCE_DIFF_MAX,
7083                            ))
7084                        }
7085                        _ => None,
7086                    };
7087                    Some(prompt::recovery_review(rec, since.as_deref()))
7088                }
7089                _ => None,
7090            };
7091
7092            // Each reviewer gets its own detached checkout of exactly this
7093            // commit: nobody can perturb the winner's tree, and the fixer can
7094            // keep working without racing a reviewer.
7095            let mut jobs = Vec::new();
7096            let coordination = self.coordinate(&repo).await?;
7097            for (r, spec) in reviewers.iter().cloned().enumerate() {
7098                let wt = root.join(format!("review-{}", r + 1));
7099                // `clean -fdx` in a reset wipes setup's products, so every
7100                // round sets the seat up again.
7101                if wt.exists() {
7102                    git::reset_detached(&wt, &head).await?;
7103                } else {
7104                    git::worktree_add_detached(&repo, &wt, &head).await?;
7105                }
7106                self.setup_seat_worktree(&repo, &wt, false).await?;
7107                let seat_key = format!("review-{}", r + 1);
7108                // The seat starts the round on whoever answered it last, not
7109                // on the agent the spec names, so a failure is not re-paid.
7110                let spec = pick_start_spec(
7111                    &self.roles.reviewer_roster,
7112                    spec,
7113                    self.state.seat_history.get(&seat_key),
7114                );
7115                let seat = self.seat(&seat_key, &spec.id);
7116                jobs.push(SeatJob {
7117                    prompt: prompt::review(&prompt::ReviewCtx {
7118                        instruction: &self.state.instruction,
7119                        branch: &winner.branch,
7120                        base_short: &base_short,
7121                        stat: &stat,
7122                        patch: &patch,
7123                        verification: prev_verification.as_ref(),
7124                        prior: prior_section.as_deref(),
7125                        reviewers: reviewers.len(),
7126                        round,
7127                        rounds: max_rounds,
7128                        // A review-only run has no rankings, so nothing
7129                        // competed for this patch and the reviewer is told so.
7130                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7131                        lens: Lens::for_seat(r),
7132                        language: &language,
7133                        github_language: &self.state.config.graph.github_language,
7134                    }),
7135                    spec,
7136                    seat,
7137                    cwd: wt,
7138                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7139                    allow_write: false,
7140                    sessions,
7141                    artifacts: artifacts.clone(),
7142                    stem: format!("review-{round}-{}", r + 1),
7143                    handover: None,
7144                });
7145            }
7146
7147            drop(coordination);
7148            self.state.event(
7149                "review",
7150                format!(
7151                    "round {round}: {} reviewers on {}",
7152                    jobs.len(),
7153                    short(&head)
7154                ),
7155            );
7156            let mut quota_losses = Vec::new();
7157            let review_retries = self.state.config.graph.retries;
7158            let review_cache = self.state.config.cache_dir();
7159            let ctx = WaveCtx {
7160                carry_seats: true,
7161                pauses: Some((&self.pause, &self.interrupt)),
7162                run: &run_id,
7163                node: "review",
7164                prompts: &prompts,
7165                cache: review_cache.as_deref(),
7166                round: Some(round),
7167            };
7168            let results = ask_json_wave::<Review>(
7169                jobs,
7170                Arc::clone(&self.sem),
7171                review_retries,
7172                &self.roles.reviewer_roster,
7173                &ctx,
7174                &mut quota_losses,
7175                &mut self.state,
7176                &|_: &Review| Ok(()),
7177            )
7178            .await;
7179            if self.state.resource_block.is_some() {
7180                return self.stop_for_resource();
7181            }
7182            // Counted before the move below: how many of *this* round's
7183            // reviewer seats were lost to their own rate limit, as opposed to
7184            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7185            let round_quota_missing = quota_losses.len();
7186            self.state.quota.extend(quota_losses);
7187
7188            let mut records = Vec::new();
7189            let mut all_findings = Vec::new();
7190            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7191                let agent_id = seat.agent.clone();
7192                self.state.seats.insert(seat.key.clone(), seat);
7193                let mut record = ReviewRecord {
7194                    reviewer: r + 1,
7195                    agent: agent_id,
7196                    summary: String::new(),
7197                    findings: Vec::new(),
7198                    vote: None,
7199                    failed: None,
7200                    duration_ms: 0,
7201                    // Set for both outcomes: `failed: Some(_)` with
7202                    // `attempts > 0` is a seat every retry still lost, not a
7203                    // recovered one — only `failed: None` with `attempts > 0`
7204                    // reads as "answered after a nudge" (see this field's own
7205                    // doc).
7206                    attempts,
7207                };
7208                match res {
7209                    Ok((review, out)) => {
7210                        // Sanitized here, at the point every other piece of
7211                        // agent prose in this file is (candidate summaries,
7212                        // deliberation turns, vote reasons): a reviewer's own
7213                        // words are the one thing about it that could name
7214                        // it, and reconsideration below broadcasts this same
7215                        // summary and these same findings to every other
7216                        // seat on the panel.
7217                        record.summary =
7218                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7219                        record.vote = Some(review.vote);
7220                        record.duration_ms = out.duration_ms;
7221                        for (n, mut f) in review.findings.into_iter().enumerate() {
7222                            // ids are magi's, never the agent's: the fixer's
7223                            // adoption report is keyed by them.
7224                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7225                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7226                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7227                            // `file` is agent-supplied prose too, never
7228                            // checked against the real tree — the same
7229                            // exposure `title`/`detail` above have, just in
7230                            // a field easy to forget because it looks like a
7231                            // path rather than free text.
7232                            f.file = f
7233                                .file
7234                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7235                            all_findings.push(f.clone());
7236                            record.findings.push(f);
7237                        }
7238                        self.state.event(
7239                            "review",
7240                            format!(
7241                                "round {round}: reviewer {} voted {} with {} finding(s)",
7242                                r + 1,
7243                                review.vote.label(),
7244                                record.findings.len()
7245                            ),
7246                        );
7247                    }
7248                    Err(e) => {
7249                        record.failed = Some(e.to_string());
7250                        self.state.event(
7251                            "review",
7252                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7253                        );
7254                    }
7255                }
7256                records.push(record);
7257            }
7258
7259            // Tally the round's votes and, if they split, spend the one
7260            // round of reconsideration the split -> deliberate -> revote
7261            // shape `judge`/`vote` use for the panel, sized down to what a
7262            // read-only review round can afford: one round, and a revote
7263            // rather than an argument, because the panel already wrote its
7264            // reasoning down as findings the first time around.
7265            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7266            let vote_split =
7267                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7268            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7269            if vote_split {
7270                self.state.event(
7271                    "review",
7272                    format!(
7273                        "round {round}: votes split ({}) — one round of reconsideration",
7274                        initial_votes
7275                            .iter()
7276                            .map(|v| v.label())
7277                            .collect::<Vec<_>>()
7278                            .join(", ")
7279                    ),
7280                );
7281                // Seats read every seat's findings and votes, still numbered
7282                // and never named — the same anonymity `review` itself keeps.
7283                let panel: Vec<ReviewSeatReport<'_>> = records
7284                    .iter()
7285                    .filter_map(|r| {
7286                        r.vote.map(|vote| ReviewSeatReport {
7287                            reviewer: r.reviewer,
7288                            vote,
7289                            summary: &r.summary,
7290                            findings: &r.findings,
7291                        })
7292                    })
7293                    .collect();
7294
7295                let mut jobs = Vec::new();
7296                let mut seats_at = Vec::new();
7297                for (r, spec) in reviewers.iter().cloned().enumerate() {
7298                    // A seat with no initial vote has nothing to reconsider
7299                    // from and stays absent, the same as it stayed absent
7300                    // from `panel` above.
7301                    if records[r].vote.is_none() {
7302                        continue;
7303                    }
7304                    let wt = root.join(format!("review-{}", r + 1));
7305                    let seat_key = format!("review-{}", r + 1);
7306                    let spec = self.occupant(&seat_key, spec);
7307                    let seat = self.seat(&seat_key, &spec.id);
7308                    // A seat with no live session has already forgotten the
7309                    // initial review's prompt — restate the patch it is
7310                    // voting on, the same as `deliberate`/`vote` do for a
7311                    // judge in the same position.
7312                    // The panel already carries this seat's own review and
7313                    // vote, so restating the patch makes the prompt whole for
7314                    // a seat handed to another agent.
7315                    let build = |with_patch: bool| {
7316                        prompt::review_reconsider(&ReviewReconsiderCtx {
7317                            instruction: &self.state.instruction,
7318                            reviewer: r + 1,
7319                            lens: Lens::for_seat(r),
7320                            panel: &panel,
7321                            patch: with_patch.then_some(ReviewPatch {
7322                                branch: &winner.branch,
7323                                base_short: &base_short,
7324                                stat: &stat,
7325                                patch: &patch,
7326                            }),
7327                            round,
7328                            rounds: max_rounds,
7329                            language: &language,
7330                        })
7331                    };
7332                    let full = build(true);
7333                    let prompt = if has_context(&spec, &seat, sessions) {
7334                        build(false)
7335                    } else {
7336                        full.clone()
7337                    };
7338                    jobs.push(SeatJob {
7339                        prompt,
7340                        spec,
7341                        seat,
7342                        cwd: wt,
7343                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7344                        allow_write: false,
7345                        sessions,
7346                        artifacts: artifacts.clone(),
7347                        stem: format!("review-{round}-reconsider-{}", r + 1),
7348                        handover: Some(full),
7349                    });
7350                    seats_at.push(r);
7351                }
7352
7353                let mut recon_quota_losses = Vec::new();
7354                let recon_cache = self.state.config.cache_dir();
7355                let recon_ctx = WaveCtx {
7356                    carry_seats: true,
7357                    pauses: Some((&self.pause, &self.interrupt)),
7358                    run: &run_id,
7359                    node: "review",
7360                    prompts: &prompts,
7361                    cache: recon_cache.as_deref(),
7362                    round: Some(round),
7363                };
7364                let recon_results = ask_json_wave::<ReviewRevote>(
7365                    jobs,
7366                    Arc::clone(&self.sem),
7367                    review_retries,
7368                    &self.roles.reviewer_roster,
7369                    &recon_ctx,
7370                    &mut recon_quota_losses,
7371                    &mut self.state,
7372                    &|_: &ReviewRevote| Ok(()),
7373                )
7374                .await;
7375                if self.state.resource_block.is_some() {
7376                    return self.stop_for_resource();
7377                }
7378                self.state.quota.extend(recon_quota_losses);
7379
7380                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7381                    let agent_id = seat.agent.clone();
7382                    self.state.seats.insert(seat.key.clone(), seat);
7383                    let mut rec = ReviewRevoteRecord {
7384                        reviewer: r + 1,
7385                        agent: agent_id,
7386                        vote: None,
7387                        reason: String::new(),
7388                        failed: None,
7389                    };
7390                    match res {
7391                        Ok((rv, _)) => {
7392                            rec.vote = Some(rv.vote);
7393                            rec.reason =
7394                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7395                            self.state.event(
7396                                "review",
7397                                format!(
7398                                    "round {round}: reviewer {} revoted {}",
7399                                    r + 1,
7400                                    rv.vote.label()
7401                                ),
7402                            );
7403                        }
7404                        Err(e) => {
7405                            rec.failed = Some(e.to_string());
7406                            self.state.event(
7407                                "review",
7408                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7409                            );
7410                        }
7411                    }
7412                    reconsideration.push(rec);
7413                }
7414            } else if initial_votes.len() > 1 {
7415                self.state.event(
7416                    "review",
7417                    format!(
7418                        "round {round}: votes agreed ({}) — no reconsideration",
7419                        initial_votes[0].label()
7420                    ),
7421                );
7422            }
7423
7424            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7425            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7426            // A round that already has a blocking finding and a round left to
7427            // try is going back to the fixer no matter what `verify.e2e`
7428            // says, so running it first only spends the loop's slowest step
7429            // (minutes, for a Rust repo's full test suite) on a head about
7430            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7431            // runs once a round has no blocking findings left (see
7432            // `round_is_clean`, which a deferred — empty — `e2e` can never
7433            // satisfy since `blocking` is nonzero whenever this branch is
7434            // taken), and `stop_reviewing` forces a real run before it will
7435            // ever read a deferred round as green.
7436            let defer_e2e =
7437                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7438            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7439                let reason =
7440                    format!("{blocking} blocking finding(s) already required a fix this round");
7441                self.state.event(
7442                    "verify",
7443                    format!(
7444                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7445                         {}); it will run once a round has none left",
7446                        short(&head)
7447                    ),
7448                );
7449                (Vec::new(), false, true, Some(reason))
7450            } else {
7451                let e2e_commands = self.state.config.verify.e2e.clone();
7452                let cache_dir = self.state.config.cache_dir();
7453                let context = format!("round {round}");
7454                let wait = self.repository_wait();
7455                let (e2e, verify_retried) = with_cache_lease(
7456                    &mut self.state,
7457                    cache_dir.as_deref(),
7458                    "e2e",
7459                    "e2e",
7460                    &winner.worktree,
7461                    &head,
7462                    verify_timeout,
7463                    &context,
7464                    &wait,
7465                    |state, budget| {
7466                        let shell = shell.clone();
7467                        let e2e_commands = e2e_commands.clone();
7468                        let worktree = winner.worktree.clone();
7469                        let context = context.clone();
7470                        async move {
7471                            run_e2e_with_retry(
7472                                state,
7473                                &shell,
7474                                &e2e_commands,
7475                                &worktree,
7476                                budget,
7477                                &context,
7478                            )
7479                            .await
7480                        }
7481                    },
7482                )
7483                .await?;
7484                (e2e, verify_retried, false, None)
7485            };
7486
7487            let expected = records.len();
7488            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7489            let incomplete = answered < expected;
7490            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7491            let policy = self.state.config.graph.incomplete_review;
7492            let clean = round_is_clean(
7493                blocking,
7494                e2e_ok,
7495                answered,
7496                expected,
7497                round_quota_missing,
7498                policy,
7499            );
7500
7501            let mut round_record = ReviewRound {
7502                round,
7503                head: head.clone(),
7504                verified_head: None,
7505                verified_at: None,
7506                reviews: records,
7507                e2e,
7508                verify_retried,
7509                e2e_deferred,
7510                e2e_defer_reason,
7511                fix: None,
7512                blocking,
7513                answered,
7514                expected,
7515                clean,
7516                progressed: false,
7517                vote_split,
7518                reconsideration,
7519                verdict: None,
7520            };
7521            // The final vote per seat is its revote where reconsideration
7522            // ran and answered, its initial vote otherwise — the same
7523            // fallback `tally` uses for a judge whose private vote failed.
7524            round_record.verdict = ReviewVote::worst(
7525                round_record
7526                    .final_votes()
7527                    .into_iter()
7528                    .map(|(_, _, vote)| vote),
7529            );
7530            // Which commit and when magi actually attempted to check —
7531            // known the moment a command was dispatched against `head`,
7532            // whether or not it finished: a resource-blocked attempt still
7533            // targeted a specific commit at a specific time, and leaving
7534            // that unrecorded is exactly what made `verification_summary`
7535            // report a fresh attempt as "commit unknown ... recorded before
7536            // this was tracked", indistinguishable from a genuinely old,
7537            // untracked record. Only a deferred or unconfigured round never
7538            // ran at all and has nothing to record — see
7539            // `ReviewRound::verified_head`'s own doc.
7540            if !matches!(
7541                round_record.e2e_status(),
7542                E2eStatus::Deferred | E2eStatus::NotConfigured
7543            ) {
7544                round_record.verified_head = Some(head.clone());
7545                round_record.verified_at = Some(Timestamp::now());
7546            }
7547
7548            if incomplete {
7549                let missing: Vec<String> = round_record
7550                    .reviews
7551                    .iter()
7552                    .filter(|r| r.failed.is_some())
7553                    .map(|r| format!("review-{}", r.reviewer))
7554                    .collect();
7555                self.state.event(
7556                    "review",
7557                    format!(
7558                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7559                        missing.join(", ")
7560                    ),
7561                );
7562            }
7563
7564            if clean {
7565                self.state.event(
7566                    "review",
7567                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7568                        format!(
7569                            "round {round}: clean (warn policy, incomplete panel) — no \
7570                             blocking findings from the seats that answered, verification green"
7571                        )
7572                    } else if incomplete {
7573                        format!(
7574                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7575                             quorum) — no blocking findings from the seats that answered, \
7576                             verification green",
7577                            expected - answered
7578                        )
7579                    } else {
7580                        format!("round {round}: clean — no blocking findings, verification green")
7581                    },
7582                );
7583                self.state.reviews.push(round_record);
7584                self.state.status = RunStatus::Gating;
7585                self.state.save()?;
7586                return Ok(());
7587            }
7588
7589            // Nothing was raised and verification passed, but not every seat
7590            // answered and `round_is_clean` still refused to call it clean —
7591            // either a seat is missing for a reason other than its own quota
7592            // (a crash, a timeout, unparsable output — worth another try), or
7593            // every seat that could have answered lost its quota and nobody
7594            // is left to decide on: re-review rather than send the fixer
7595            // after a round with nothing to fix.
7596            if incomplete && blocking == 0 && e2e_ok {
7597                self.state.reviews.push(round_record);
7598                self.state.save()?;
7599                if round == max_rounds {
7600                    self.state.status = RunStatus::Blocked;
7601                    self.state.event(
7602                        "review",
7603                        format!(
7604                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7605                             refusing to call it clean",
7606                            expected - answered
7607                        ),
7608                    );
7609                    return Ok(());
7610                }
7611                continue;
7612            }
7613
7614            // Nothing for the fixer to act on (`blocking == 0`) and the only
7615            // reason this round is not clean is that magi itself never got
7616            // a command to run — the shared build cache, not the patch (see
7617            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7618            // the fixer would invite a change to appease contention that has
7619            // nothing to do with the diff, and would leave this attempt
7620            // sitting in the next round's prompt as if it were about an
7621            // earlier, superseded commit rather than what it actually is:
7622            // the same head, still waiting to be checked. Wait for it the
7623            // same way the final round's own contention is already handled,
7624            // whatever round this happens to be.
7625            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7626                self.state.reviews.push(round_record);
7627                return self
7628                    .stop_unresolved_review(
7629                        "the round's own verification could not run",
7630                        &shell,
7631                        &winner.worktree,
7632                    )
7633                    .await;
7634            }
7635
7636            if round == max_rounds {
7637                self.state.reviews.push(round_record);
7638                return self
7639                    .stop_reviewing(
7640                        &format!(
7641                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7642                        ),
7643                        &shell,
7644                        &winner.worktree,
7645                    )
7646                    .await;
7647            }
7648
7649            if !self.fix_review_round(&winner, round_record).await? {
7650                return Ok(());
7651            }
7652        }
7653        Ok(())
7654    }
7655
7656    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7657    /// False means admission or continuation policy stopped the review loop.
7658    async fn fix_review_round(
7659        &mut self,
7660        winner: &Candidate,
7661        mut round_record: ReviewRound,
7662    ) -> Result<bool> {
7663        let round = round_record.round;
7664        let shell = self.state.config.shell();
7665        let step = self.run_fix_round(winner, &mut round_record).await?;
7666        if matches!(step, FixStep::Resource) {
7667            self.stop_for_resource()?;
7668            return Ok(false);
7669        }
7670        if self.state.reviews.len() < round {
7671            self.state.reviews.push(round_record);
7672        } else {
7673            self.state.reviews[round - 1] = round_record;
7674        }
7675        self.state.save()?;
7676
7677        // The fixer's own report never came back this round, even after
7678        // `continue_fix_report`'s own budget was spent on it — not an
7679        // ordinary "no report" (dropped stream, quota, plain failure),
7680        // which already reads that way and is left to the existing round
7681        // budget. Stopping here, rather than opening another round, is
7682        // what keeps a next reviewer/fixer wave from ever being
7683        // dispatched onto `winner.worktree` while whatever the seat's
7684        // last call may still have running there is unaccounted for: no
7685        // process liveness check exists (and none is being added — see
7686        // AGENTS.md/this task's own scope), so the only way to honour
7687        // "nothing starts before a valid report returns" is to not start
7688        // anything further on this worktree from this run at all.
7689        if matches!(step, FixStep::Recorded { report_lost: true }) {
7690            self.stop_reviewing(
7691                "the fixer's adoption report never came back, even after resuming its \
7692                         own seat; refusing to start another round against the same worktree \
7693                         while that is unresolved",
7694                &shell,
7695                &winner.worktree,
7696            )
7697            .await?;
7698            return Ok(false);
7699        }
7700
7701        let streak = self
7702            .state
7703            .reviews
7704            .iter()
7705            .rev()
7706            .take_while(|r| !r.progressed)
7707            .count();
7708        if streak >= STAGNANT_LIMIT {
7709            self.stop_reviewing(
7710                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7711                &shell,
7712                &winner.worktree,
7713            )
7714            .await?;
7715            return Ok(false);
7716        }
7717        Ok(true)
7718    }
7719
7720    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7721    async fn run_fix_round(
7722        &mut self,
7723        winner: &Candidate,
7724        round_record: &mut ReviewRound,
7725    ) -> Result<FixStep> {
7726        let round = round_record.round;
7727        let max_rounds = self.state.config.graph.review_rounds;
7728        let run_id = self.state.id.clone();
7729        let prompts = self.state.config.prompts.clone();
7730        let language = self.state.config.graph.language.clone();
7731        let sessions = self.state.config.graph.sessions;
7732        let artifacts = self.merge_repair_artifacts();
7733        let base = self.landing_base();
7734        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7735        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7736        let this_round_verification = round_record.verification_summary(&head);
7737        let all_findings: Vec<_> = round_record
7738            .reviews
7739            .iter()
7740            .flat_map(|r| r.findings.iter().cloned())
7741            .collect();
7742        // Fix. The winner's own implementer seat continues its conversation:
7743        // the competition is over, so context is pure benefit now.
7744        let blocking_findings: Vec<_> = all_findings
7745            .iter()
7746            .filter(|f| f.severity.blocks())
7747            .cloned()
7748            .collect();
7749        let fix_prompt = prompt::fix_in(
7750            &self.state.instruction,
7751            &blocking_findings,
7752            this_round_verification.as_ref(),
7753            round,
7754            max_rounds,
7755            &language,
7756            &self.state.config.graph.github_language,
7757        );
7758        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7759        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7760        let (job, seat, out) = self
7761            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7762                prompt: fix_prompt.clone(),
7763                spec,
7764                seat,
7765                cwd: winner.worktree.clone(),
7766                timeout,
7767                allow_write: true,
7768                sessions,
7769                artifacts: artifacts.clone(),
7770                stem: format!("fix-{round}"),
7771                handover: None,
7772            })
7773            .await;
7774        // Not asked: the round is not recorded (so a resume repeats it),
7775        // no fixer was handed over to, and the seat is left as it was.
7776        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7777            return Ok(FixStep::Resource);
7778        }
7779        let agent_id = seat.agent.clone();
7780
7781        let mut fix = FixRecord {
7782            agent: agent_id,
7783            addressed: Vec::new(),
7784            rejected: Vec::new(),
7785            notes: String::new(),
7786            committed: false,
7787            failed: None,
7788            duration_ms: 0,
7789            continuation: None,
7790        };
7791        let mut continuation = ContinuationRecord::not_needed();
7792        let mut final_seat = seat.clone();
7793        match out {
7794            AgentOutcome::Ok(o) => {
7795                fix.duration_ms = o.duration_ms;
7796                let parsed = verdict::extract_json::<FixReport>(&o.text);
7797                // A parsed report standing next to a command this same
7798                // reply's own CLI never confirmed the exit status of is
7799                // not a resolved answer — the identical `CommandEvidence`
7800                // `state.jobs` renders, read here instead of only on
7801                // display, per the completion judgment and the shown
7802                // record needing to agree.
7803                let incomplete_reason = match &parsed {
7804                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7805                        "the reply parsed, but it reported a command whose own CLI never \
7806                             confirmed an exit status"
7807                            .to_owned(),
7808                    ),
7809                    Ok(_) => None,
7810                    Err(e) => Some(e.to_string()),
7811                };
7812                match incomplete_reason {
7813                    None => {
7814                        let report = parsed.expect("checked Ok above");
7815                        fix.addressed = report.addressed;
7816                        fix.rejected = report.rejected;
7817                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7818                    }
7819                    Some(reason) => {
7820                        let (resumed_seat, resolved, failure, cont) = self
7821                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7822                            .await;
7823                        fix.duration_ms += cont.cumulative_wait_ms;
7824                        continuation = cont;
7825                        final_seat = resumed_seat;
7826                        match resolved {
7827                            Some(report) => {
7828                                fix.addressed = report.addressed;
7829                                fix.rejected = report.rejected;
7830                                fix.notes =
7831                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7832                            }
7833                            None => fix.failed = failure,
7834                        }
7835                    }
7836                }
7837            }
7838            // The CLI's raw error JSON is not a fix report to parse.
7839            AgentOutcome::Dropped(o) => {
7840                fix.duration_ms = o.duration_ms;
7841                let why = o
7842                    .dropped
7843                    .as_ref()
7844                    .map(|d| d.why.as_str())
7845                    .unwrap_or("the CLI ended the stream without delivering its answer");
7846                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7847            }
7848            AgentOutcome::Quota(o) => {
7849                self.state.quota.push(QuotaLoss::from_report(
7850                    final_seat.key.clone(),
7851                    "fix",
7852                    o.quota.as_ref(),
7853                ));
7854                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7855            }
7856            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7857        }
7858        if self.state.resource_block.is_some() {
7859            self.state.seats.insert(final_seat.key.clone(), final_seat);
7860            return Ok(FixStep::Resource);
7861        }
7862        fix.continuation = Some(continuation);
7863        self.state.seats.insert(final_seat.key.clone(), final_seat);
7864        let coordination = self.coordinate(&winner.worktree).await?;
7865        if let Ok(r) = git::rescue_commit(
7866            &winner.worktree,
7867            &format!("magi: review round {round} fixes (uncommitted work)"),
7868        )
7869        .await
7870        {
7871            self.state.note_withheld("fix", &r.withheld);
7872        }
7873        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7874        fix.committed = after != before;
7875        // Judged by what `git` says moved against base, never by the
7876        // fixer's own `addressed`/`rejected` count — see
7877        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7878        // `patch` snapshot above: swallowing this error would default
7879        // `diff_after` to empty, which almost always differs from a
7880        // non-empty `patch` and reads as "progressed" — exactly backwards
7881        // for a `git` failure the stagnation check cannot see through.
7882        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7883        drop(coordination);
7884        let progressed = diff_after != patch;
7885        let commit_note = if fix.committed {
7886            "committed"
7887        } else {
7888            "NO new commit"
7889        };
7890        let tree_note = if progressed {
7891            "changed vs base"
7892        } else {
7893            "unchanged vs base"
7894        };
7895        self.state.event(
7896            "fix",
7897            match &fix.failed {
7898                // Distinct on purpose from "0 addressed, 0 rejected": the
7899                // fixer's own diff still landed (blocking counts do keep
7900                // falling round over round), only its adoption report did
7901                // not come back, so this must never read like every
7902                // finding was reviewed and declined.
7903                Some(reason) => {
7904                    format!(
7905                        "round {round}: fixer's adoption report was lost ({reason}); \
7906                             {commit_note}, tree {tree_note}"
7907                    )
7908                }
7909                None => format!(
7910                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7911                         {tree_note}{}",
7912                    fix.addressed.len(),
7913                    fix.rejected.len(),
7914                    if continuation.outcome == ContinuationOutcome::Resumed {
7915                        format!(
7916                            " (adoption report recovered after {} continuation(s))",
7917                            continuation.attempts
7918                        )
7919                    } else {
7920                        String::new()
7921                    },
7922                ),
7923            },
7924        );
7925        round_record.fix = Some(fix);
7926        round_record.progressed = progressed;
7927        Ok(FixStep::Recorded {
7928            report_lost: matches!(
7929                continuation.outcome,
7930                ContinuationOutcome::Exhausted
7931                    | ContinuationOutcome::QuotaLost
7932                    | ContinuationOutcome::NoSession
7933            ),
7934        })
7935    }
7936
7937    /// Decide, from the last recorded round's own verification, whether
7938    /// stopping the review loop is a hand-off or a genuine block.
7939    ///
7940    /// Called once the loop has given up trying — the round budget is spent,
7941    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
7942    /// findings still open, never while a round is still clean or the
7943    /// incomplete-panel case handled inline above. Gate and e2e are facts
7944    /// about the tree; a lingering review finding is an opinion, and this
7945    /// workload's own `magi stats` puts reviewer precision low enough
7946    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
7947    /// must not by itself stand between a green, verified change and the
7948    /// human who decides what to do with it. A red e2e is not an opinion, so
7949    /// that case still blocks, with the failing command and a tail of its
7950    /// output recorded here rather than left in `run.json` for someone to go
7951    /// find.
7952    ///
7953    /// A round that deferred its own e2e (see [`Config::graph`]'s
7954    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
7955    /// only because nothing ran, and treating an empty list as a passing one
7956    /// here is exactly the "deferred painted green" bug this function exists
7957    /// to not have. When the last round's own verification never resolved —
7958    /// deferred on purpose, or a real attempt the shared build cache blocked
7959    /// — this makes (or retries) the real run, on the actual worktree this
7960    /// loop is about to stop touching, before deciding anything. A
7961    /// resource-blocked attempt is likewise never read as either green or
7962    /// red: it is evidence about the machine, not the patch (see
7963    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
7964    /// blocked cache leaves this call without deciding rather than guessing
7965    /// — the caller retries on a later reentry.
7966    /// Record, once, that the review loop handed off over a blocking finding
7967    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
7968    /// `land` asks the owner even with `land_approval` off. Called from every
7969    /// path that concludes `Gating`; a reentry keeps the first record.
7970    fn record_contested_handoff(&mut self) {
7971        if self.state.contested_handoff.is_some() {
7972            return;
7973        }
7974        let Some(contested) = self
7975            .state
7976            .reviews
7977            .last()
7978            .and_then(ReviewRound::contested_handoff)
7979        else {
7980            return;
7981        };
7982        self.state.event(
7983            "review",
7984            format!(
7985                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
7986                 wait for the owner's approval",
7987                contested.findings.len(),
7988                contested.rejecters.len()
7989            ),
7990        );
7991        self.state.contested_handoff = Some(contested);
7992    }
7993
7994    async fn stop_unresolved_review(
7995        &mut self,
7996        why: &str,
7997        shell: &[String],
7998        worktree: &Path,
7999    ) -> Result<()> {
8000        self.stop_reviewing_inner(why, shell, worktree, true).await
8001    }
8002
8003    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8004        self.stop_reviewing_inner(why, shell, worktree, false).await
8005    }
8006
8007    async fn stop_reviewing_inner(
8008        &mut self,
8009        why: &str,
8010        shell: &[String],
8011        worktree: &Path,
8012        repairable: bool,
8013    ) -> Result<()> {
8014        let round_idx = self.state.reviews.len() - 1;
8015        // A deferred round and a resource-blocked one are the same shape
8016        // here: neither has a real result yet, and both get one more
8017        // attempt. Read off `e2e_status` — the single source for this —
8018        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8019        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8020        // still retries instead of being read as a settled result the
8021        // instant it stops being empty.
8022        let needs_catchup_run = matches!(
8023            self.state.reviews[round_idx].e2e_status(),
8024            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8025        );
8026        if needs_catchup_run {
8027            let round = self.state.reviews[round_idx].round;
8028            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8029            let commands = self.state.config.verify.e2e.clone();
8030            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8031            let cache_dir = self.state.config.cache_dir();
8032            let context = format!(
8033                "round {round}: verification unresolved, catching up before the final decision"
8034            );
8035            let wait = self.repository_wait();
8036            let (outcomes, verify_retried) = with_cache_lease(
8037                &mut self.state,
8038                cache_dir.as_deref(),
8039                "e2e",
8040                "e2e",
8041                worktree,
8042                &attempted_head,
8043                timeout,
8044                &context,
8045                &wait,
8046                |state, budget| {
8047                    let shell = shell.to_vec();
8048                    let commands = commands.clone();
8049                    let context = context.clone();
8050                    async move {
8051                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8052                            .await
8053                    }
8054                },
8055            )
8056            .await?;
8057            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8058                self.state.record_base_progress(&attempted_head);
8059            }
8060            let last = &mut self.state.reviews[round_idx];
8061            last.e2e = outcomes;
8062            last.verify_retried = verify_retried;
8063            // Always the commit and time this attempt actually targeted,
8064            // whether or not it happens to equal the reviewed `head` and
8065            // whether or not a command finished — see
8066            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8067            // attempt is recorded too, so a later reader sees "attempted
8068            // again at T2" rather than silence.
8069            last.verified_head = Some(attempted_head);
8070            last.verified_at = Some(Timestamp::now());
8071            if verify_inconclusive(&last.e2e) {
8072                // Still not a real result: `e2e_deferred` is left exactly
8073                // as it was, so `needs_catchup_run` above reads
8074                // `ResourceBlocked` (via `e2e_status`, which checks
8075                // `resource_blocked` before `e2e_deferred`) and retries
8076                // again on the next reentry, rather than recording
8077                // contention as a red e2e and blocking the run on it.
8078                self.state.save()?;
8079                return Ok(());
8080            }
8081            last.e2e_deferred = false;
8082            // A complete panel may have been saved as unclean solely because
8083            // verification could not run. Refresh that verdict with the new
8084            // outcomes before a granted repair consumes the round's result.
8085            // Incomplete panels retain their original policy/quota decision.
8086            if last.answered == last.expected {
8087                last.clean = round_is_clean(
8088                    last.blocking,
8089                    last.e2e.iter().all(CommandOutcome::ok),
8090                    last.answered,
8091                    last.expected,
8092                    0,
8093                    self.state.config.graph.incomplete_review,
8094                );
8095            }
8096        }
8097        let last = &self.state.reviews[round_idx];
8098        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8099
8100        match last.e2e_status() {
8101            E2eStatus::Failed => {
8102                let red: Vec<String> = last
8103                    .e2e
8104                    .iter()
8105                    .filter(|o| !o.ok())
8106                    .map(|o| {
8107                        format!(
8108                            "`{}` -> {:?}\n{}",
8109                            o.command,
8110                            o.code,
8111                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8112                        )
8113                    })
8114                    .collect();
8115                let can_repair = last.blocking == 0 && last.fix.is_none();
8116                self.state.reviews[round_idx].clean = false;
8117                self.state
8118                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8119                self.state.status = RunStatus::Blocked;
8120                if repairable
8121                    && can_repair
8122                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8123                {
8124                    self.state.status = RunStatus::Reviewing;
8125                }
8126            }
8127            // `needs_catchup_run` above already retried once this call; if
8128            // it is still blocked, this is magi's own admission it could
8129            // not get a command to run, never a verdict on the patch — the
8130            // run is left exactly where a later reentry can retry again.
8131            E2eStatus::ResourceBlocked => {
8132                self.state.event(
8133                    "review",
8134                    format!(
8135                        "{why}; e2e could not run (resource unavailable); not \
8136                         deciding yet"
8137                    ),
8138                );
8139            }
8140            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8141                self.state.event(
8142                    "review",
8143                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8144                );
8145                self.record_contested_handoff();
8146                self.state.status = RunStatus::Gating;
8147            }
8148        }
8149        self.state.save()?;
8150        Ok(())
8151    }
8152
8153    // ----------------------------------------------------------------- gate
8154
8155    async fn gate(&mut self) -> Result<()> {
8156        if self.hold_base_sync().await? {
8157            return Ok(());
8158        }
8159        // Judged by the review record itself, not by `status`: a solo
8160        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8161        // reentry (see `judge`), and trusting it here is exactly how a run
8162        // that exhausted its review budget got gated and merged a second
8163        // time around. `review_conclusion` recomputes the review loop's own
8164        // verdict from the round records themselves — `Gating` for a clean
8165        // round or a hand-off (see `stop_reviewing`), anything else means the
8166        // loop is still going or genuinely blocked.
8167        // A base the winner could not be replayed onto is a decision, not a
8168        // round: there is no landing tree to gate. Read as its own record for
8169        // the same reason the review verdict is.
8170        if self.state.status == RunStatus::Failed
8171            || self
8172                .state
8173                .base_sync
8174                .as_ref()
8175                .is_some_and(|s| s.conflict.is_some())
8176            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8177                != Some(RunStatus::Gating)
8178        {
8179            return Ok(());
8180        }
8181        let Some(winner) = self.state.winner().cloned() else {
8182            return Ok(());
8183        };
8184        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8185        self.state.reconcile_verification_head(&head);
8186        self.state.save()?;
8187        let wait = self.repository_wait();
8188        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8189            return Ok(());
8190        }
8191        if self.state.gate_ran {
8192            // `review_loop` derives its conclusion from the clean review
8193            // record on every reentry and therefore puts a completed run back
8194            // in `Gating`. A recorded gate is a stronger, terminal fact:
8195            // retain its original command output (or lack of any, for a repo
8196            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8197            // doc) and restore `Blocked` on a real failure rather than
8198            // pretending the command is still running or running it a second
8199            // time. `gate_ran == false` remains the only shape — unattempted,
8200            // or a resource-blocked retry — that may still need to execute a
8201            // command.
8202            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8203                self.state.status = RunStatus::Blocked;
8204                self.state.save()?;
8205            }
8206            return Ok(());
8207        }
8208        self.state.status = RunStatus::Gating;
8209        let mut outcomes = self.run_gate(&winner).await?;
8210        let mut attempted_head = self.state.gate_head.clone();
8211        loop {
8212            // A resource-blocked outcome means the gate command never actually
8213            // ran - the shared build cache could not be acquired or confirmed
8214            // fresh in time - which is evidence about the machine, not about
8215            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8216            // Recording it as a red gate would mark a run `Blocked` on nothing
8217            // but contention magi has already logged; leaving `self.state.gate`
8218            // empty and `self.state.gate_ran` false instead keeps the shape
8219            // this function already treats as "still needs to run" (see the
8220            // early-return above), so the next call retries the command
8221            // rather than concluding anything.
8222            if verify_inconclusive(&outcomes) {
8223                self.state.save()?;
8224                return Ok(());
8225            }
8226            if outcomes.iter().all(CommandOutcome::ok) {
8227                break;
8228            }
8229            let fixes_before = self.state.gate_fixes.len();
8230            match self.gate_fix_round(&winner, &outcomes).await? {
8231                GateFix::Retry => {
8232                    outcomes = self.run_gate(&winner).await?;
8233                    attempted_head = self.state.gate_head.clone();
8234                }
8235                GateFix::Stop => {
8236                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8237                    if self.state.gate_fixes.len() > fixes_before
8238                        && attempted_head.as_deref() != Some(current.as_str())
8239                        && verify_current_head(&mut self.state, &winner, &wait).await?
8240                    {
8241                        // A new SHA owes a gate even when a spent fixer changed only
8242                        // commit metadata. This does not grant another repair round.
8243                        outcomes = self.run_gate(&winner).await?;
8244                        attempted_head = self.state.gate_head.clone();
8245                    } else {
8246                        break;
8247                    }
8248                }
8249                GateFix::Defer => {
8250                    self.state.save()?;
8251                    return Ok(());
8252                }
8253            }
8254        }
8255        let passed = outcomes.iter().all(CommandOutcome::ok);
8256        self.state.gate = outcomes;
8257        self.state.gate_ran = true;
8258        self.state.gate_head = attempted_head;
8259        if !self.state.gate.is_empty()
8260            && !verify_inconclusive(&self.state.gate)
8261            && let Some(head) = self.state.gate_head.clone()
8262        {
8263            self.state.record_base_progress(&head);
8264        }
8265        self.state.save()?;
8266        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8267            return Ok(());
8268        }
8269        if !passed {
8270            self.state.status = RunStatus::Blocked;
8271            let spent = self.state.gate_fixes.len();
8272            self.state.event(
8273                "gate",
8274                if spent == 0 {
8275                    "gate failed; not merging".to_owned()
8276                } else {
8277                    format!("gate failed after {spent} gate-fix round(s); not merging")
8278                },
8279            );
8280        }
8281        self.state.save()?;
8282        Ok(())
8283    }
8284
8285    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8286    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8287    /// after review is clean and never on a candidate awaiting judging.
8288    ///
8289    /// Never fails the run: a non-zero exit or timeout is a warning and a
8290    /// recorded outcome, and the gate remains the single arbiter. Nothing
8291    /// configured means nothing happens - no event, no commit. `commit_all`
8292    /// commits any leftover change under the neutral identity and returns
8293    /// `false` when the tree is clean, so no empty commit is ever made.
8294    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8295        let commands = state.config.verify.pre_gate.clone();
8296        if commands.is_empty() {
8297            return;
8298        }
8299        let shell = state.config.shell();
8300        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8301        let (outcomes, _) = run_commands(
8302            state,
8303            "pre_gate",
8304            "pre_gate",
8305            0,
8306            &shell,
8307            &commands,
8308            &winner.worktree,
8309            timeout,
8310        )
8311        .await;
8312        for o in &outcomes {
8313            if !o.ok() {
8314                tracing::warn!(
8315                    "pre_gate `{}` failed ({:?}); the gate decides",
8316                    o.command,
8317                    o.code
8318                );
8319            }
8320            state.event(
8321                "pre_gate",
8322                format!(
8323                    "`{}` -> {}",
8324                    o.command,
8325                    if o.ok() {
8326                        "pass".to_owned()
8327                    } else {
8328                        format!(
8329                            "FAIL ({:?})\n{}",
8330                            o.code,
8331                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8332                        )
8333                    }
8334                ),
8335            );
8336        }
8337        state.pre_gate = outcomes;
8338        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8339            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8340                Ok(head) => {
8341                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8342                    state.pre_gate_commit = Some(head);
8343                }
8344                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8345            },
8346            Ok(false) => {}
8347            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8348        }
8349        if let Err(e) = state.save() {
8350            tracing::warn!("could not persist the pre_gate record: {e:#}");
8351        }
8352    }
8353
8354    /// Run `verify.gate` once against the winner's current tree, logging one
8355    /// event per command. Empty when nothing is configured.
8356    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8357        let wait = self.repository_wait();
8358        run_landing_gate(&mut self.state, winner, &wait).await
8359    }
8360
8361    /// One bounded fix round for a failing gate.
8362    ///
8363    /// The fixer is told the failure came from the gate itself, not from a
8364    /// reviewer, and is shown the failed commands, their exit codes and a tail
8365    /// of their output - whatever `[verify].gate` holds, nothing here knows
8366    /// what those commands run. Only a normal non-zero exit that printed
8367    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8368    /// command or a full disk says nothing about the code, and a fixer sent
8369    /// after it can only appease the machine. The round is judged by what git
8370    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8371    /// again before the gate does, so a fix cannot trade a green gate for a
8372    /// red e2e unnoticed.
8373    async fn gate_fix_round(
8374        &mut self,
8375        winner: &Candidate,
8376        outcomes: &[CommandOutcome],
8377    ) -> Result<GateFix> {
8378        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8379            g.phase == crate::repair_budget::Phase::Fixing
8380                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8381        }) {
8382            self.state.gate_fixes.len().saturating_add(1)
8383        } else {
8384            self.state.config.graph.gate_fix_rounds
8385        };
8386        let spent = self.state.gate_fixes.len();
8387        if spent >= cap {
8388            if cap > 0 {
8389                self.state.event(
8390                    "gate",
8391                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8392                );
8393            }
8394            return Ok(GateFix::Stop);
8395        }
8396        if !gate_fixable(outcomes) {
8397            self.state.event(
8398                "gate",
8399                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8400                 command or similar); not spending a fix round on it",
8401            );
8402            return Ok(GateFix::Stop);
8403        }
8404        let min_free = self.state.config.disk.min_free_bytes;
8405        if min_free > 0 {
8406            match crate::disk::free_bytes(&winner.worktree) {
8407                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8408                Ok(free) => {
8409                    self.state.event(
8410                        "gate",
8411                        format!(
8412                            "only {free} bytes free ({min_free} required by `[disk] \
8413                             min_free_bytes`); not spending a fix round on a failure the disk \
8414                             may explain"
8415                        ),
8416                    );
8417                    return Ok(GateFix::Stop);
8418                }
8419                Err(e) => {
8420                    self.state.event(
8421                        "gate",
8422                        format!("free disk space could not be measured ({e:#}); no fix round"),
8423                    );
8424                    return Ok(GateFix::Stop);
8425                }
8426            }
8427        }
8428
8429        let attempt = spent + 1;
8430        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8431        let base = self.landing_base();
8432        let fix_prompt = prompt::gate_fix_in(
8433            &self.state.instruction,
8434            &failed,
8435            attempt,
8436            cap,
8437            &self.state.config.graph.language,
8438            &self.state.config.graph.github_language,
8439        );
8440        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8441        let sessions = self.state.config.graph.sessions;
8442        let artifacts = self.merge_repair_artifacts();
8443        self.state.event(
8444            "gate",
8445            format!("gate failed; gate-fix round {attempt} of {cap}"),
8446        );
8447        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8448        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8449        // Persist before invoking the fixer: it may commit and then be
8450        // interrupted before its result (or the following e2e) is saved.
8451        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8452            head: before.clone(),
8453            passed: false,
8454            outcomes: Vec::new(),
8455        });
8456        self.state.save()?;
8457        let (_, seat, out) = self
8458            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8459                prompt: fix_prompt.clone(),
8460                spec,
8461                seat,
8462                cwd: winner.worktree.clone(),
8463                timeout,
8464                allow_write: true,
8465                sessions,
8466                artifacts: artifacts.clone(),
8467                stem: format!("gate-fix-{attempt}"),
8468                handover: None,
8469            })
8470            .await;
8471        // The fixer was never asked: no round is spent (nothing is recorded)
8472        // and a later reentry asks the same agent again.
8473        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8474            return Ok(GateFix::Defer);
8475        }
8476        let mut record = GateFixRecord {
8477            agent: seat.agent.clone(),
8478            failed,
8479            notes: String::new(),
8480            committed: false,
8481            error: None,
8482        };
8483        match out {
8484            AgentOutcome::Ok(o) => {
8485                // A missing report is not a failed fix: the round is judged
8486                // by the tree below, and the report only carries prose.
8487                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8488                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8489                }
8490            }
8491            AgentOutcome::Dropped(_) => {
8492                record.error = Some("the CLI dropped the stream".to_owned());
8493            }
8494            AgentOutcome::Quota(o) => {
8495                self.state.quota.push(QuotaLoss::from_report(
8496                    seat.key.clone(),
8497                    "gate-fix",
8498                    o.quota.as_ref(),
8499                ));
8500                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8501            }
8502            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8503        }
8504        self.state.seats.insert(seat.key.clone(), seat);
8505        let coordination = self.coordinate(&winner.worktree).await?;
8506        if let Ok(r) = git::rescue_commit(
8507            &winner.worktree,
8508            &format!("magi: gate fix {attempt} (uncommitted work)"),
8509        )
8510        .await
8511        {
8512            self.state.note_withheld("gate-fix", &r.withheld);
8513        }
8514        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8515        record.committed = after != before;
8516        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8517        let note = record.error.clone();
8518        self.state.gate_fixes.push(record);
8519        self.state.save()?;
8520        if !changed {
8521            self.state.event(
8522                "gate",
8523                match note {
8524                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8525                    None => format!("gate-fix round {attempt}: the tree did not change"),
8526                },
8527            );
8528            return Ok(GateFix::Stop);
8529        }
8530        self.state.event(
8531            "gate",
8532            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8533        );
8534
8535        drop(coordination);
8536        let wait = self.repository_wait();
8537        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8538    }
8539
8540    // ---------------------------------------------------------------- merge
8541
8542    /// One read-only rewrite by the summary's author; a text the gate still
8543    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8544    /// applies on their word or on silence. `None` means the run is parked on
8545    /// that question and nothing may be pushed or opened yet.
8546    async fn guarded_pr_message(
8547        &mut self,
8548        winner: &Candidate,
8549        facts: Option<&BranchFacts>,
8550        posting: bool,
8551    ) -> Result<Option<PrMessage>> {
8552        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8553        let head = match final_head.as_deref() {
8554            Some(h) => crate::pr_outcome::FinalHead::At(h),
8555            None => crate::pr_outcome::FinalHead::Unknown,
8556        };
8557        // Only a pull request that is about to be posted pays for a summary.
8558        let summarized = if posting {
8559            Some(self.summarize_change(winner, final_head.as_deref()).await)
8560        } else {
8561            None
8562        };
8563        // A busy shared resource is not a permanent cause: publishing an
8564        // "unavailable" description would stick (a later head move never
8565        // edits the body). Stop before anything is pushed or opened; the
8566        // run is `Blocked` and a resume asks the summary again.
8567        if self.state.resource_block.is_some() {
8568            self.state.event(
8569                "pr-summary",
8570                "a shared resource was busy; nothing was published, resume to summarize again",
8571            );
8572            return Ok(None);
8573        }
8574        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8575            anyhow::bail!(
8576                "the final commit moved during public description generation; retry before publishing"
8577            );
8578        }
8579        let note = match (&summarized, &self.state.change_summary) {
8580            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8581            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8582            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8583            (None, _) => ChangeNote::NotAsked,
8584        };
8585        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8586        if !posting {
8587            let identity = crate::scrub::Identity::current();
8588            return Ok(Some(PrMessage {
8589                title: crate::scrub::scrub(&pr.title, &identity),
8590                body: crate::scrub::scrub(&pr.body, &identity),
8591            }));
8592        }
8593        // This very text was already rejected and asked about: no second
8594        // judge call, rewrite or question, just carry the record on.
8595        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8596        if let Some(known) = self
8597            .state
8598            .github_text
8599            .clone()
8600            .filter(|g| g.fingerprint == raw_fp)
8601        {
8602            return self.settle_withheld(winner, &pr, known).await;
8603        }
8604        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8605        let mut violations =
8606            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8607        let identity = crate::scrub::Identity::current();
8608        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8609            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8610            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8611        {
8612            violations.push(crate::github_text::Violation::SensitiveData);
8613        }
8614        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8615        // only a language violation starts one; sensitive data alone goes
8616        // straight to the owner, with the redacted text as the fallback.
8617        let has_language = violations
8618            .iter()
8619            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8620        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8621            self.state.event(
8622                "github-text",
8623                if has_language {
8624                    format!("description rejected: {violations:?}; requesting one rewrite")
8625                } else {
8626                    format!("description rejected: {violations:?}; asking the owner")
8627                },
8628            );
8629            let mut rewritten = false;
8630            let mut rejected_rewrite = None;
8631            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8632            {
8633                let output = text;
8634                if let Some(title) = summary_title(&output) {
8635                    // Judge the redacted candidate that will actually be posted.
8636                    // Sensitive data alone must not discard a useful rewrite.
8637                    let raw_body = summary_without_title(&output);
8638                    let clean_title = crate::scrub::scrub(&title, &identity);
8639                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8640                    let redacted = clean_title != title || body != raw_body;
8641                    let title = clean_title;
8642                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8643                    if !complete {
8644                        self.state.event(
8645                            "github-text",
8646                            "description rewrite omitted required findings",
8647                        );
8648                    }
8649                    if !body.trim().is_empty() {
8650                        body.push_str("\n\n");
8651                        body.push_str(&public_pr_evidence(&self.state, head));
8652                    }
8653                    let marker = format!("magi:run/{}", self.state.id);
8654                    if !body.trim().is_empty() && !body.contains(&marker) {
8655                        body.push_str(&format!("\n\n{marker}"));
8656                    }
8657                    let rewrite = PrMessage {
8658                        title: title.clone(),
8659                        body: body.clone(),
8660                    };
8661                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8662                    let mut failed = crate::github_text::check_for(
8663                        &self.state.config.graph,
8664                        &title,
8665                        &body,
8666                        redecision,
8667                    );
8668                    if (!crate::github_text::shareable(&title)
8669                        || !crate::github_text::shareable(&body))
8670                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8671                    {
8672                        failed.push(crate::github_text::Violation::SensitiveData);
8673                    }
8674                    if complete
8675                        && !title.trim().is_empty()
8676                        && !body.trim().is_empty()
8677                        && !failed.is_empty()
8678                    {
8679                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8680                            title: title.clone(),
8681                            body: body.clone(),
8682                            failed: failed
8683                                .iter()
8684                                .map(|v| crate::github_text::category(*v).to_owned())
8685                                .collect(),
8686                        });
8687                    }
8688                    if complete
8689                        && !title.trim().is_empty()
8690                        && !raw_body.trim().is_empty()
8691                        && failed.is_empty()
8692                    {
8693                        decision = redecision;
8694                        pr = PrMessage { title, body };
8695                        rewritten = true;
8696                        self.state
8697                            .event("github-text", "description rewrite passed");
8698                        if redacted {
8699                            self.state.event(
8700                                "github-text",
8701                                "sensitive data removed from description rewrite",
8702                            );
8703                        }
8704                    }
8705                }
8706            }
8707            if !rewritten {
8708                self.state.event(
8709                    "github-text",
8710                    "description rewrite unavailable or rejected; asking the owner",
8711                );
8712                if let Some(w) =
8713                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8714                {
8715                    // A different text than the one asked about before: that
8716                    // question no longer describes anything.
8717                    self.retire_github_text_question("the withheld text changed");
8718                    let record = crate::run::GithubTextAsk {
8719                        rewrite: rejected_rewrite,
8720                        fingerprint: raw_fp,
8721                        title: w.title,
8722                        body: w.body,
8723                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8724                        question: None,
8725                        asks: 0,
8726                        resolved: false,
8727                        chosen_title: None,
8728                        choice: None,
8729                    };
8730                    return self.settle_withheld(winner, &pr, record).await;
8731                }
8732            }
8733        }
8734        let (title, body) =
8735            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8736        Ok(Some(PrMessage { title, body }))
8737    }
8738
8739    /// Describe the whole change from its final diff in either run mode.
8740    ///
8741    /// The summary is tied to the final commit and to the merge base of the
8742    /// resolved remote base; a record for any other pair is dropped and the
8743    /// agent asked again, so a description never outlives the diff it was
8744    /// written from. The seat is read-only and the prompt tells it to read the
8745    /// diff and the code around it. `[roles] synthesizer` selects the agents
8746    /// (unset: the default pick), each asked at most once, advancing on an
8747    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8748    /// carries the reason it is unavailable; nothing is invented then.
8749    async fn summarize_change(
8750        &mut self,
8751        winner: &Candidate,
8752        head: Option<&str>,
8753    ) -> std::result::Result<(), String> {
8754        let Some(head) = head else {
8755            return Err("the final commit could not be read".to_owned());
8756        };
8757        let remote = self.state.config.merge.remote.clone();
8758        let tracking = format!("{remote}/{}", self.state.base_branch);
8759        if !git::rev_exists(&winner.worktree, &tracking).await {
8760            return Err(format!("the remote base `{tracking}` could not be read"));
8761        }
8762        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8763            Ok(b) if !b.is_empty() => b,
8764            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8765        };
8766        let evidence = crate::pr_outcome::summary_evidence(
8767            &self.state,
8768            crate::pr_outcome::FinalHead::At(head),
8769        );
8770        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8771        if self
8772            .state
8773            .change_summary
8774            .as_ref()
8775            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8776        {
8777            return Ok(());
8778        }
8779        if self.state.change_summary.take().is_some() {
8780            self.state.event(
8781                "pr-summary",
8782                "the final diff or evidence changed; the earlier change summary was discarded",
8783            );
8784        }
8785        // Save the source snapshot even if no generation agent is available.
8786        if let Err(e) = write_artifact(
8787            &self.state,
8788            &format!("public-description-{fingerprint}.md"),
8789            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8790        ) {
8791            tracing::warn!("could not save public description evidence: {e:#}");
8792        }
8793        let chain = match agent::pick_chain(
8794            &self.state.config.agents,
8795            self.state.config.roles.synthesizer.as_ref(),
8796            &agent::installed,
8797            "synthesizer",
8798        ) {
8799            Ok(chain) => chain,
8800            Err(e) => {
8801                tracing::warn!("no agent for the change summary: {e:#}");
8802                self.state.event("pr-summary", "no summary agent available");
8803                return Err("no agent was available to summarize it".to_owned());
8804            }
8805        };
8806        let subjects = git::subjects(&winner.worktree, &base, head)
8807            .await
8808            .unwrap_or_default();
8809        let stat = git::diff_stat(&winner.worktree, &base, head)
8810            .await
8811            .unwrap_or_default();
8812        let stat: String = stat.chars().take(6_000).collect();
8813        let mut prompt = crate::prompt::change_summary_in(
8814            &base,
8815            head,
8816            &subjects,
8817            stat.trim(),
8818            &self.state.config.graph.github_language,
8819        );
8820        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8821        let sessions = self.state.config.graph.sessions;
8822        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8823        let prompts = self.state.config.prompts.clone();
8824        let cache = self.state.config.cache_dir();
8825        let run = self.state.id.clone();
8826        for (i, spec) in chain.into_iter().enumerate() {
8827            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8828            let github_spec = match agent::for_github_text(
8829                &spec,
8830                &self.state.config.graph.github_language,
8831                &winner.worktree,
8832            ) {
8833                Ok(spec) => spec,
8834                Err(e) => {
8835                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8836                    continue;
8837                }
8838            };
8839            let job = SeatJob {
8840                spec: github_spec,
8841                seat,
8842                cwd: winner.worktree.clone(),
8843                prompt: prompt.clone(),
8844                timeout,
8845                allow_write: false,
8846                sessions,
8847                artifacts: agent::artifacts_dir(&self.state.dir()),
8848                stem: if i == 0 {
8849                    "change-summary".to_owned()
8850                } else {
8851                    format!("change-summary-{}", spec.id)
8852                },
8853                handover: None,
8854            };
8855            let ctx = WaveCtx {
8856                run: &run,
8857                node: "pr-summary",
8858                prompts: &prompts,
8859                cache: cache.as_deref(),
8860                round: None,
8861                carry_seats: false,
8862                pauses: Some((&self.pause, &self.interrupt)),
8863            };
8864            let (seat, outcome) =
8865                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8866            self.state.seats.insert(seat.key.clone(), seat);
8867            let text = match outcome {
8868                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8869                AgentOutcome::ResourceBlocked(why) => {
8870                    // No agent was asked: the run stops resumably before
8871                    // anything is published (see `guarded_pr_message`).
8872                    if self.state.resource_block.is_none() {
8873                        self.state.resource_block = Some(crate::run::ResourceBlock {
8874                            cause: crate::run::ResourceBlockCause::Other,
8875                            node: "pr-summary".to_owned(),
8876                            kind: "shared resource".to_owned(),
8877                            detail: why,
8878                            at: Timestamp::now(),
8879                        });
8880                    }
8881                    return Err("a shared resource was busy".to_owned());
8882                }
8883                _ => continue,
8884            };
8885            let blind = &self.state.config.blind;
8886            let Some(title) = summary_title(&text) else {
8887                continue;
8888            };
8889            let body: String = summary_without_title(&text)
8890                .lines()
8891                .filter(|l| !l.contains("magi:run/"))
8892                .collect::<Vec<_>>()
8893                .join("\n");
8894            let body = blind::sanitize_prose(body.trim(), blind);
8895            let title = blind::sanitize_prose(&title, blind);
8896            if body.trim().is_empty() || title.trim().is_empty() {
8897                continue;
8898            }
8899            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8900                continue;
8901            }
8902            // The branch must still be the one the agent was shown.
8903            if git::rev_parse(&winner.worktree, "HEAD")
8904                .await
8905                .ok()
8906                .as_deref()
8907                != Some(head)
8908            {
8909                return Err("the branch moved while the summary was written".to_owned());
8910            }
8911            if git::merge_base(&winner.worktree, &tracking, head)
8912                .await
8913                .ok()
8914                .as_deref()
8915                != Some(base.as_str())
8916                || crate::pr_outcome::summary_evidence(
8917                    &self.state,
8918                    crate::pr_outcome::FinalHead::At(head),
8919                ) != evidence
8920            {
8921                return Err("the summary evidence changed while it was written".to_owned());
8922            }
8923            self.state.change_summary = Some(crate::run::ChangeSummary {
8924                head: head.to_owned(),
8925                base,
8926                evidence: fingerprint,
8927                title,
8928                body,
8929            });
8930            self.state.event(
8931                "pr-summary",
8932                format!("change summary written by {} for the final diff", spec.id),
8933            );
8934            if let Err(e) = self.state.save() {
8935                tracing::warn!("could not save the change summary: {e:#}");
8936            }
8937            return Ok(());
8938        }
8939        self.state
8940            .event("pr-summary", "no agent produced a change summary");
8941        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
8942    }
8943
8944    /// One read-only rewrite request for a withheld description. The
8945    /// implementer's own conversation is tried first when it exists and can be
8946    /// resumed; then the `[roles] language_judge` chain (or the default pick)
8947    /// writes it in a separate conversation, skipping the implementer's agent.
8948    /// The prompt carries the whole text, so no agent needs any memory. Each
8949    /// agent is asked at most once and the chain advances on an error, quota,
8950    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
8951    /// that has run out of quota must not end the rewrite. Nothing is recorded
8952    /// as a handover.
8953    async fn rewrite_pr_text(
8954        &mut self,
8955        winner: &Candidate,
8956        pr: &PrMessage,
8957        violations: &[crate::github_text::Violation],
8958    ) -> Option<String> {
8959        let prompt = format!(
8960            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
8961            pr.title, pr.body
8962        );
8963        let sessions = self.state.config.graph.sessions;
8964        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
8965        // (spec, the resumed implementer's seat). A fallback seat is created
8966        // only when its turn comes, so an unused fallback leaves no record.
8967        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
8968        let key = format!("impl-{}", winner.label);
8969        if let Some(spec) = self
8970            .state
8971            .config
8972            .agents
8973            .iter()
8974            .find(|a| a.id == winner.agent)
8975            .cloned()
8976            && let Some(existing) = self.state.seats.get(&key)
8977            && existing.agent == spec.id
8978            && agent::has_session(spec.kind, existing, sessions)
8979        {
8980            attempts.push((spec, Some(existing.clone())));
8981        }
8982        match agent::pick_chain(
8983            &self.state.config.agents,
8984            self.state.config.roles.language_judge.as_ref(),
8985            &agent::installed,
8986            "language_judge",
8987        ) {
8988            Ok(chain) => {
8989                for spec in chain {
8990                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
8991                        continue;
8992                    }
8993                    attempts.push((spec, None));
8994                }
8995            }
8996            Err(e) if attempts.is_empty() => {
8997                tracing::warn!("no agent for the description rewrite: {e:#}");
8998                self.state
8999                    .event("github-text", "no rewrite agent available");
9000                return None;
9001            }
9002            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9003        }
9004        let prompts = self.state.config.prompts.clone();
9005        let cache = self.state.config.cache_dir();
9006        let run = self.state.id.clone();
9007        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9008            let resumed = resumed_seat.is_some();
9009            let seat = resumed_seat.unwrap_or_else(|| {
9010                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9011            });
9012            let github_spec = match agent::for_github_text(
9013                &spec,
9014                &self.state.config.graph.github_language,
9015                &winner.worktree,
9016            ) {
9017                Ok(spec) => spec,
9018                Err(e) => {
9019                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9020                    continue;
9021                }
9022            };
9023            let job = SeatJob {
9024                spec: github_spec,
9025                seat,
9026                cwd: winner.worktree.clone(),
9027                prompt: prompt.clone(),
9028                timeout: retry_budget(timeout_full, resumed),
9029                allow_write: false,
9030                sessions,
9031                artifacts: agent::artifacts_dir(&self.state.dir()),
9032                stem: if i == 0 {
9033                    "github-text-rewrite".to_owned()
9034                } else {
9035                    format!("github-text-rewrite-{}", spec.id)
9036                },
9037                handover: None,
9038            };
9039            let ctx = WaveCtx {
9040                run: &run,
9041                node: "github-text",
9042                prompts: &prompts,
9043                cache: cache.as_deref(),
9044                round: None,
9045                carry_seats: false,
9046                pauses: Some((&self.pause, &self.interrupt)),
9047            };
9048            let (seat, outcome) =
9049                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9050            self.state.seats.insert(seat.key.clone(), seat);
9051            match outcome {
9052                AgentOutcome::Ok(o)
9053                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9054                {
9055                    return Some(o.text);
9056                }
9057                AgentOutcome::ResourceBlocked(_) => return None,
9058                _ => {}
9059            }
9060        }
9061        None
9062    }
9063
9064    /// Abandon the standing posting-gate question, if any.
9065    fn retire_github_text_question(&mut self, why: &str) {
9066        let Some(id) = self
9067            .state
9068            .github_text
9069            .as_ref()
9070            .and_then(|g| g.question.clone())
9071        else {
9072            return;
9073        };
9074        let _ = ask::Questions::open().update(&id, |q| {
9075            q.abandon(why);
9076            Ok(())
9077        });
9078    }
9079
9080    /// Carry a withheld text to its end: file the question, park while it is
9081    /// open, and on an answer, silence or timeout pick the text to post. The
9082    /// decision is saved before it is used, so it applies once.
9083    async fn settle_withheld(
9084        &mut self,
9085        winner: &Candidate,
9086        pr: &PrMessage,
9087        mut g: crate::run::GithubTextAsk,
9088    ) -> Result<Option<PrMessage>> {
9089        use crate::github_text as gt;
9090        let store = ask::Questions::open();
9091        let timeout = self.state.config.graph.answer_timeout;
9092        if !g.resolved {
9093            let mut standing = g
9094                .question
9095                .as_ref()
9096                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9097            if let Some(q) = standing.as_ref()
9098                && q.status.open()
9099                && gt::expired(q, timeout)
9100            {
9101                let why = format!("no answer within {}s of asking", timeout.max(1));
9102                standing = store
9103                    .update(&q.id, |q| {
9104                        q.abandon(&why);
9105                        Ok(())
9106                    })
9107                    .ok()
9108                    .map(|(q, ())| q);
9109            }
9110            let mut ask_again: Option<Option<Vec<String>>> = None;
9111            match standing {
9112                // Recorded but never written (a stop between the two saves),
9113                // or removed: ask again without spending the retry.
9114                None if g.asks <= 1 => {
9115                    g.asks = 0;
9116                    ask_again = Some(None);
9117                }
9118                None => {
9119                    g.resolved = true;
9120                    g.chosen_title = None;
9121                    g.choice = Some(GithubTextChoice::Fallback);
9122                }
9123                Some(q) if q.status.open() => {
9124                    self.park_on_github_text(&g)?;
9125                    return Ok(None);
9126                }
9127                Some(q) => match gt::read_reply(&q) {
9128                    gt::Reply::Fallback => {
9129                        g.resolved = true;
9130                        g.chosen_title = None;
9131                        g.choice = Some(GithubTextChoice::Fallback);
9132                    }
9133                    gt::Reply::Original => {
9134                        g.resolved = true;
9135                        g.chosen_title = None;
9136                        g.choice = Some(GithubTextChoice::Original);
9137                    }
9138                    gt::Reply::Rewrite => {
9139                        g.resolved = true;
9140                        g.chosen_title = None;
9141                        g.choice = Some(if g.rewrite.is_some() {
9142                            GithubTextChoice::Rewrite
9143                        } else {
9144                            GithubTextChoice::Fallback
9145                        });
9146                    }
9147                    gt::Reply::Title(text) => {
9148                        let first = text
9149                            .lines()
9150                            .map(str::trim)
9151                            .find(|l| !l.is_empty())
9152                            .unwrap_or("");
9153                        let decision = if first.is_empty() {
9154                            None
9155                        } else {
9156                            let candidate = PrMessage {
9157                                title: first.to_owned(),
9158                                body: String::new(),
9159                            };
9160                            self.judge_pr_language(&winner.worktree, &candidate).await
9161                        };
9162                        match gt::vet_title(
9163                            &text,
9164                            decision,
9165                            self.state.config.graph.github_english(),
9166                        ) {
9167                            Ok(title) => {
9168                                g.resolved = true;
9169                                g.chosen_title = Some(title);
9170                                g.choice = Some(GithubTextChoice::MyTitle);
9171                            }
9172                            Err(cats) if g.asks < 2 => {
9173                                ask_again = Some(Some(
9174                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9175                                ))
9176                            }
9177                            Err(_) => {
9178                                g.resolved = true;
9179                                g.chosen_title = None;
9180                                g.choice = Some(GithubTextChoice::Fallback);
9181                            }
9182                        }
9183                    }
9184                },
9185            }
9186            if let Some(rejected) = ask_again {
9187                // A retry (after a rejected replacement title) is still about
9188                // the original text: the same ORIGINAL / FALLBACK comparison.
9189                let w = gt::Withheld {
9190                    title: g.title,
9191                    body: g.body,
9192                    categories: g.categories.clone(),
9193                };
9194                // Named by the original text's fingerprint alone, so a resume
9195                // rewrites the same file and the detail stays identical.
9196                let artifact = match crate::run::write_artifact(
9197                    &self.state,
9198                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9199                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9200                ) {
9201                    Ok(p) => Some(p),
9202                    Err(e) => {
9203                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9204                        None
9205                    }
9206                };
9207                let (detail, comparison) = gt::question_content(
9208                    &w,
9209                    &pr.title,
9210                    &pr.body,
9211                    &self.state.id,
9212                    rejected.as_deref(),
9213                    artifact.as_deref(),
9214                    g.rewrite.as_ref(),
9215                );
9216                let mut q = ask::Question::new(
9217                    self.state.id.clone(),
9218                    gt::ASK_NODE.to_owned(),
9219                    gt::ASK_SEAT.to_owned(),
9220                    gt::question_summary(&self.state.config.graph.language, &w),
9221                    detail,
9222                    gt::question_choices(&w, g.rewrite.as_ref()),
9223                );
9224                q.posting_comparison = Some(comparison);
9225                q.answer_timeout = timeout;
9226                // Recorded before it is written: a stop in between leaves a
9227                // record naming a question the resume then files again.
9228                g.question = Some(q.id.clone());
9229                g.asks += 1;
9230                self.state.github_text = Some(g.clone());
9231                self.state.event(
9232                    "github-text",
9233                    format!("asking the owner about the withheld text ({})", q.short()),
9234                );
9235                self.state.save()?;
9236                store
9237                    .put_executable(&mut q)
9238                    .context("file the posting-gate question")?;
9239                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9240                    tracing::warn!(
9241                        "could not notify about posting-gate question {}: {e:#}",
9242                        q.short()
9243                    );
9244                }
9245                self.park_on_github_text(&g)?;
9246                return Ok(None);
9247            }
9248        }
9249        let resolved = g.chosen_title.clone();
9250        let original = g.choice == Some(GithubTextChoice::Original);
9251        // Field names only: the event log never records the text.
9252        self.state.event(
9253            "github-text",
9254            if original {
9255                format!(
9256                    "posting the original text for: {}",
9257                    match (g.title, g.body) {
9258                        (true, true) => "title, description",
9259                        (true, false) => "title",
9260                        _ => "description",
9261                    }
9262                )
9263            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9264                "posting the saved rewrite title and description".to_owned()
9265            } else if resolved.is_some() {
9266                "posting the owner's replacement title".to_owned()
9267            } else {
9268                "posting the neutral text for the withheld fields".to_owned()
9269            },
9270        );
9271        self.state.github_text = Some(g.clone());
9272        self.state.save()?;
9273        if g.choice == Some(GithubTextChoice::Rewrite)
9274            && let Some(r) = g.rewrite.as_ref()
9275        {
9276            return Ok(Some(PrMessage {
9277                title: r.title.clone(),
9278                body: r.body.clone(),
9279            }));
9280        }
9281        let w = gt::Withheld {
9282            title: g.title,
9283            body: g.body,
9284            categories: g.categories.clone(),
9285        };
9286        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9287        if let Some(t) = resolved {
9288            title = t;
9289        }
9290        // Every withheld field is now neutral or vetted English.
9291        let english = gt::LanguageDecision {
9292            title_english: true,
9293            body_english: true,
9294            body_complete: true,
9295        };
9296        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9297        if original {
9298            // The owner's explicit choice overrides the gate for the fields it
9299            // was made for; the others keep what the gate produced.
9300            if g.title {
9301                title = pr.title.clone();
9302            }
9303            if g.body {
9304                body = pr.body.clone();
9305            }
9306        }
9307        Ok(Some(PrMessage { title, body }))
9308    }
9309
9310    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9311        self.state.github_text = Some(g.clone());
9312        self.state.end_pass_parked(ParkCause::Question);
9313        self.state.parked = true;
9314        self.state.event(
9315            "github-text",
9316            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9317        );
9318        self.state.save()
9319    }
9320
9321    /// Ask `[roles] language_judge` about this text and record which source
9322    /// decided; the text itself is never recorded.
9323    async fn judge_pr_language(
9324        &mut self,
9325        cwd: &Path,
9326        pr: &PrMessage,
9327    ) -> Option<crate::github_text::LanguageDecision> {
9328        self.state.config.roles.language_judge.as_ref()?;
9329        if !self.state.config.graph.language_gate() {
9330            return None;
9331        }
9332        let decision =
9333            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9334        self.state.event(
9335            "github-text",
9336            if decision.is_some() {
9337                "language decided by the language judge"
9338            } else {
9339                "language judge unavailable; using built-in heuristics"
9340            },
9341        );
9342        decision
9343    }
9344
9345    async fn merge(&mut self) -> Result<()> {
9346        if self.resource_halt()? || self.hold_base_sync().await? {
9347            return Ok(());
9348        }
9349        // Same reasoning as `gate`: ask the review and gate records directly
9350        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9351        // skip can rewrite on reentry to something that no longer says
9352        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9353        // so a hand-off (open findings, green verification) reaches merge
9354        // exactly like a genuinely clean round does.
9355        //
9356        // A run resumed mid-`land` never reaches here at all: `execute`
9357        // recognises `RunStatus::Landing` before it even calls `prep`, and
9358        // routes straight to `run_land` instead. That has to happen a level
9359        // up from this function, not with a check in here, because
9360        // `review_loop`'s own status recomputation (see its doc) runs
9361        // *before* `merge` on every reentry and would otherwise overwrite
9362        // the `Landing` marker with `Gating` before this node ever saw it.
9363        if self
9364            .state
9365            .base_sync
9366            .as_ref()
9367            .is_some_and(|s| s.conflict.is_some())
9368            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9369                != Some(RunStatus::Gating)
9370            // `gate_ran == false` is not "passed" - `gate` leaves it false
9371            // both before it has ever run and when its last attempt was
9372            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9373            // permission to merge on nothing but the review record. Only a
9374            // gate that actually ran may proceed; an intentional local
9375            // review can also finish with verification explicitly not configured.
9376            // `RunState::gate_status` computes that distinction.
9377            || !self.state.gate_allows_progress()
9378        {
9379            return Ok(());
9380        }
9381        // This node's own record, not `status`: `status == Ready` is not
9382        // unique to the harmless `MergeMode::None` path this line was
9383        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9384        // run's PR was closed without merging — and on that run `mode` is
9385        // still `Pr`, so a reentry that fell through here would push and
9386        // open a second pull request. `self.state.merge` is set exactly once
9387        // this node (or `land`) has already produced a verdict, under every
9388        // mode, which is what "already done" actually means here.
9389        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9390            return Ok(());
9391        }
9392        let Some(winner) = self.state.winner().cloned() else {
9393            return Ok(());
9394        };
9395        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9396        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9397            self.state.status = RunStatus::Blocked;
9398            self.state.event(
9399                "gate",
9400                "winner changed after gate; fresh verification required",
9401            );
9402            self.state.save()?;
9403            return Ok(());
9404        }
9405        let wait = self.repository_wait();
9406        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9407            return Ok(());
9408        }
9409        let repo = self.state.repo.clone();
9410        let base = self.state.base_branch.clone();
9411        let mode = self.state.config.merge.mode;
9412        let style = self.state.config.merge.style;
9413        let facts = if is_review_run(&self.state) {
9414            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9415            let start = review_base(
9416                &repo,
9417                &self.state.config.merge.remote,
9418                &base,
9419                &self.state.base_commit,
9420                &winner.branch,
9421            )
9422            .await;
9423            branch_facts(&repo, &start, &winner.branch).await
9424        } else {
9425            None
9426        };
9427        // `None` when the base could not be freshly read: then an adopted
9428        // pull request's title is left alone.
9429        let leaked = if is_review_run(&self.state) {
9430            leaked_subjects(&self.state, &winner.branch).await
9431        } else {
9432            Some(Vec::new())
9433        };
9434        // `None`: parked on the owner's word about a withheld title or body.
9435        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9436        // the resume arrives here again.
9437        let Some(pr) = self
9438            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9439            .await?
9440        else {
9441            return Ok(());
9442        };
9443        let message = pr.commit_message();
9444
9445        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9446        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9447            return Ok(());
9448        }
9449        let outcome = match mode {
9450            MergeMode::None => MergeOutcome {
9451                mode,
9452                ok: true,
9453                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9454                empty: false,
9455            },
9456            MergeMode::Pr | MergeMode::Local
9457                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9458            {
9459                MergeOutcome {
9460                    mode,
9461                    ok: false,
9462                    detail: empty_candidate_detail(&self.state, &base),
9463                    empty: true,
9464                }
9465            }
9466            MergeMode::Local => {
9467                let on = git::current_branch(&repo).await?;
9468                if on.as_deref() != Some(base.as_str()) {
9469                    MergeOutcome {
9470                        mode,
9471                        ok: false,
9472                        detail: format!(
9473                            "{} has {} checked out, not the base branch {base}",
9474                            repo.display(),
9475                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9476                        ),
9477                        empty: false,
9478                    }
9479                } else if !git::is_clean(&repo).await? {
9480                    MergeOutcome {
9481                        mode,
9482                        ok: false,
9483                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9484                        empty: false,
9485                    }
9486                } else {
9487                    let out = match style {
9488                        MergeStyle::Merge => {
9489                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9490                        }
9491                        MergeStyle::Squash => {
9492                            git::merge_squash(&repo, &winner.branch, &message).await?
9493                        }
9494                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9495                    };
9496                    MergeOutcome {
9497                        mode,
9498                        ok: out.ok(),
9499                        detail: if out.ok() { out.stdout } else { out.stderr },
9500                        empty: false,
9501                    }
9502                }
9503            }
9504            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9505                self.state.pr_publication_pending = true;
9506                self.state.save()?;
9507                let out =
9508                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9509                self.state.pr_publication_pending = out.is_err();
9510                MergeOutcome {
9511                    mode,
9512                    ok: out.is_ok(),
9513                    detail: match out {
9514                        Ok(url) => url,
9515                        Err(e) => format!("{e:#}"),
9516                    },
9517                    empty: false,
9518                }
9519            }
9520            MergeMode::Pr => {
9521                let remote = self.state.config.merge.remote.clone();
9522                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9523                let pushed = if self.state.foreign_checkout.is_some() {
9524                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9525                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9526                } else {
9527                    git::push(&winner.worktree, &remote, &winner.branch).await?
9528                };
9529                if !pushed.ok() {
9530                    MergeOutcome {
9531                        mode,
9532                        ok: false,
9533                        detail: pushed.stderr,
9534                        empty: false,
9535                    }
9536                } else {
9537                    if self.state.foreign_checkout.is_some() {
9538                        self.state.pr_pushed_head = Some(delivery_head.clone());
9539                    }
9540                    // A retry or resume of a run whose branch already has an
9541                    // open pull request adopts it rather than failing on a
9542                    // duplicate. Only this winner branch into this base:
9543                    // `branch_for` derives the name from the run id, so a
9544                    // different run's pull request never matches.
9545                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9546                    let out = match pr_merge_plan(found) {
9547                        PrPlan::Create => {
9548                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9549                                .await
9550                        }
9551                        PrPlan::Adopt { url, title } => {
9552                            self.state
9553                                .event("merge", format!("Pr: adopted open pull request {url}"));
9554                            if self.state.github_text.as_ref().is_some_and(|g| {
9555                                g.resolved
9556                                    && g.choice == Some(GithubTextChoice::Rewrite)
9557                                    && g.rewrite.is_some()
9558                            }) {
9559                                // The owner chose the entire pair, even if the
9560                                // existing title already matches or was edited.
9561                                land::set_pr_text_verbatim(
9562                                    &winner.worktree,
9563                                    &url,
9564                                    &pr.title,
9565                                    &pr.body,
9566                                )
9567                                .await
9568                                .map(|()| url)
9569                            } else {
9570                                if title != pr.title
9571                                    && (!is_review_run(&self.state)
9572                                        || leaked
9573                                            .as_deref()
9574                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9575                                    && let Err(e) =
9576                                        if self.state.github_text.as_ref().is_some_and(|g| {
9577                                            g.resolved
9578                                                && g.title
9579                                                && g.choice == Some(GithubTextChoice::Original)
9580                                        }) {
9581                                            // The owner chose to post this title unmodified.
9582                                            land::set_pr_title_verbatim(
9583                                                &winner.worktree,
9584                                                &url,
9585                                                &pr.title,
9586                                            )
9587                                            .await
9588                                        } else {
9589                                            land::set_pr_title(
9590                                                &mut self.state,
9591                                                &winner.worktree,
9592                                                &url,
9593                                                &pr.title,
9594                                            )
9595                                            .await
9596                                        }
9597                                {
9598                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9599                                    self.state
9600                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9601                                }
9602                                Ok(url)
9603                            }
9604                        }
9605                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9606                    };
9607                    match out {
9608                        Ok(url) => MergeOutcome {
9609                            mode,
9610                            ok: true,
9611                            detail: url,
9612                            empty: false,
9613                        },
9614                        Err(e) => MergeOutcome {
9615                            mode,
9616                            ok: false,
9617                            detail: e.to_string(),
9618                            empty: false,
9619                        },
9620                    }
9621                }
9622            }
9623        };
9624
9625        self.state.status = match (mode, outcome.ok) {
9626            (MergeMode::None, _) => RunStatus::Ready,
9627            (_, true) if outcome.empty => RunStatus::Ready,
9628            (MergeMode::Pr, true) => RunStatus::Ready,
9629            (_, true) => RunStatus::Merged,
9630            (_, false) => RunStatus::Blocked,
9631        };
9632        self.state.event(
9633            "merge",
9634            format!(
9635                "{:?}: {}",
9636                mode,
9637                outcome.detail.lines().next().unwrap_or("")
9638            ),
9639        );
9640        self.state.merge = Some(outcome);
9641        self.state.save()?;
9642
9643        // The PR is open and the run would historically stop here, leaving the
9644        // operator to watch checks, feed review comments back to a fixer, and
9645        // merge. That was done by hand six times in one session before this
9646        // existed. Opt-in, because merging is the one irreversible thing magi
9647        // can do to a repository.
9648        if self.state.config.graph.land
9649            && mode == MergeMode::Pr
9650            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9651        {
9652            self.run_land().await?;
9653        }
9654        // `run_land` may have left `status` at `Landing` - still waiting on
9655        // CI or the owner's approval, not actually settled - so this has to
9656        // read whatever `status` ended up as here, not the `Merged` this
9657        // function set a few lines up.
9658        self.settle_questions();
9659        Ok(())
9660    }
9661
9662    /// Enter `land`.
9663    ///
9664    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9665    /// a resumed run's re-entry. `land::land` itself is what serialises the
9666    /// two git-mutating moments inside the loop — the rebase push and
9667    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9668    /// to hold a lock across the whole call, and doing so would serialise
9669    /// this run's CI wait against a *different* run's land-approval resume
9670    /// in the same repository, which is exactly the "must not wait on
9671    /// another task" property the daemon's slot-freeing exists to give.
9672    async fn run_land(&mut self) -> Result<()> {
9673        crate::pr_head::ensure_owner(&self.state)?;
9674        let url = self
9675            .state
9676            .pr
9677            .as_ref()
9678            .map(|pr| pr.url.clone())
9679            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9680            .unwrap_or_default();
9681        let url = url.lines().next().unwrap_or("").trim().to_owned();
9682        if !url.starts_with("http") {
9683            return Ok(());
9684        }
9685        // A land failure is not a lost run: the work is on a branch and the
9686        // pull request is open, which is exactly where a human takes over.
9687        loop {
9688            match land::land(&mut self.state, &url).await {
9689                Ok(_)
9690                    if self
9691                        .state
9692                        .merge_repairs
9693                        .last()
9694                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9695                {
9696                    if self.repair_merge_review().await? {
9697                        continue;
9698                    }
9699                }
9700                Ok(_) if self.state.resource_block.is_some() => {
9701                    return self.stop_for_resource();
9702                }
9703                Ok(pr) if self.state.parked => {
9704                    // `land` already saved the parked marker; nothing here
9705                    // overrides `status` back to a terminal value while an
9706                    // approval is still outstanding.
9707                    let _ = pr;
9708                }
9709                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9710                    self.state.status = RunStatus::Superseded;
9711                    self.state.save()?;
9712                }
9713                Ok(pr) => {
9714                    self.state.status = match pr.state {
9715                        land::PrLifecycle::Merged => RunStatus::Merged,
9716                        _ => RunStatus::Blocked,
9717                    };
9718                    // Downstream of a confirmed merge only - see
9719                    // `bump::should_release_bump`'s own doc for why this one
9720                    // check covers all three of `land`'s success paths.
9721                    // Best-effort: the run already landed, so a failure here
9722                    // (the decision call, `gh`, `cargo`) is recorded and never
9723                    // turns a landed run into a failed one.
9724                    if bump::should_release_bump(self.state.status)
9725                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9726                    {
9727                        // The event is the run's own record. Not-eligible cases
9728                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9729                        // `Err` is a bump that was tried and failed:
9730                        // `after_merge` itself raises the operator notice for
9731                        // that, whether or not a release PR exists yet.
9732                        self.state
9733                            .event("bump", format!("release bump skipped: {e:#}"));
9734                    }
9735                    // Independent of the bump, and best-effort in the same way:
9736                    // findings the merge left open become follow-up tasks.
9737                    if self.state.status == RunStatus::Merged {
9738                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9739                    }
9740                    self.state.save()?;
9741                }
9742                Err(e) => {
9743                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9744                        return Err(e);
9745                    }
9746                    self.state.status = RunStatus::Blocked;
9747                    self.state.event("land", format!("gave up: {e}"));
9748                    self.state.save()?;
9749                }
9750            }
9751            break;
9752        }
9753        Ok(())
9754    }
9755
9756    /// One repair cycle stays under the existing driver/task claim. An
9757    /// interrupted fixer is rolled back and held rather than run twice.
9758    async fn repair_merge_review(&mut self) -> Result<bool> {
9759        use crate::run::MergeRepairPhase as Phase;
9760        crate::pr_head::ensure_owner(&self.state)?;
9761        let i = self.state.merge_repairs.len() - 1;
9762        let request = self.state.merge_repairs[i].clone();
9763        if request.phase == Phase::Fixing {
9764            let restored = match self.state.winner().cloned() {
9765                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9766                None => Err(anyhow::anyhow!("repair has no winning branch")),
9767            };
9768            let why = match restored {
9769                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9770                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9771                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9772            };
9773            self.hold_merge_repair(i, why)?;
9774            return Ok(false);
9775        }
9776        // A completed repair gate is a publication checkpoint. On reentry,
9777        // reject external changes before the ordinary gate path can replace
9778        // that evidence by verifying a new head. Requested repairs still
9779        // start their own review generation below.
9780        if request.phase == Phase::Reviewing && self.state.gate_ran {
9781            let winner = self
9782                .state
9783                .winner()
9784                .context("repair has no winning branch")?;
9785            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9786            if self.state.gate_head.as_deref() != Some(head.as_str())
9787                || !git::is_clean(&winner.worktree).await?
9788            {
9789                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9790                return Ok(false);
9791            }
9792        }
9793        let failure = if i >= self.state.config.graph.land_rounds
9794            || self.state.config.graph.review_rounds == 0
9795        {
9796            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9797        } else {
9798            None
9799        };
9800        if let Some(why) = failure {
9801            self.hold_merge_repair(i, why)?;
9802            return Ok(false);
9803        }
9804        if request.phase == Phase::Requested {
9805            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9806                Ok(pr) => pr,
9807                Err(e) => {
9808                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9809                    return Ok(false);
9810                }
9811            };
9812            self.state.merge_repairs[i].phase = Phase::Fixing;
9813            self.state.land_approval = None;
9814            self.state.land_armed_head = None;
9815            self.state.save()?;
9816            // Serialize the exact records, including finding ids/details,
9817            // reconsideration reasons, final votes and verification output.
9818            let evidence = serde_json::to_string_pretty(&request)?;
9819            let reason = format!(
9820                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9821            );
9822            let budget = self.state.config.graph.land_rounds;
9823            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9824            match result {
9825                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9826                Ok(land::Fixed::Deferred) => {
9827                    // An unavailable fixer did not spend this repair request.
9828                    // Restore partial fallback work before retrying it, as for
9829                    // an interrupted repair, without closing the request.
9830                    let worktree = self
9831                        .state
9832                        .winner()
9833                        .context("repair has no winning branch")?
9834                        .worktree
9835                        .clone();
9836                    self.graft_restore(&worktree, &request.head).await?;
9837                    self.state.merge_repairs[i].phase = Phase::Requested;
9838                    self.stop_for_resource()?;
9839                    return Ok(false);
9840                }
9841                Ok(land::Fixed::Failed(why)) => {
9842                    self.hold_merge_repair(
9843                        i,
9844                        format!(
9845                            "repair failed: {why}; inspect the branch and run an explicit review"
9846                        ),
9847                    )?;
9848                    return Ok(false);
9849                }
9850                Err(e) => {
9851                    self.hold_merge_repair(
9852                        i,
9853                        format!(
9854                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9855                        ),
9856                    )?;
9857                    return Ok(false);
9858                }
9859            }
9860            // Even no change gets a real new reviewer wave. The previous
9861            // evidence remains in the request; a former clean/concluded round
9862            // cannot short-circuit the new review or gate.
9863            self.begin_merge_repair_review(i)?;
9864        }
9865        self.review_loop().await?;
9866        self.gate().await?;
9867        if self.resource_halt()? || self.state.parked {
9868            return Ok(false);
9869        }
9870        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9871            != Some(RunStatus::Gating)
9872            || !self.state.gate_allows_progress()
9873        {
9874            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9875            return Ok(false);
9876        }
9877        // Review and gate may have added commits after the initial repair.
9878        // Publish exactly their final head to the existing PR, with a lease.
9879        let winner = self
9880            .state
9881            .winner()
9882            .cloned()
9883            .context("repair has no winning branch")?;
9884        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9885        if self.state.gate_head.as_deref() != Some(after.as_str())
9886            || !git::is_clean(&winner.worktree).await?
9887        {
9888            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9889            return Ok(false);
9890        }
9891        let expected = self
9892            .state
9893            .pr_pushed_head
9894            .as_deref()
9895            .unwrap_or(&request.head)
9896            .to_owned();
9897        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9898        // Push the checked commit itself: a local ref moving after the check
9899        // must not change which commit the forge receives.
9900        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9901            return Ok(false);
9902        }
9903        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9904        let refspec = format!("{after}:refs/heads/{public}");
9905        let push = git::git_raw(
9906            &winner.worktree,
9907            &["push", &lease, &self.state.config.merge.remote, &refspec],
9908        )
9909        .await;
9910        match push {
9911            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9912            Ok(out) => {
9913                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9914                return Ok(false);
9915            }
9916            Err(e) => {
9917                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9918                return Ok(false);
9919            }
9920        }
9921        self.state.merge_repairs[i].phase = Phase::Complete;
9922        self.state.status = RunStatus::Landing;
9923        self.state.event(
9924            "repair",
9925            "new review and gate completed; fresh owner merge decision required",
9926        );
9927        self.state.save()?;
9928        Ok(true)
9929    }
9930
9931    fn merge_repair_artifacts(&self) -> PathBuf {
9932        let artifacts = agent::artifacts_dir(&self.state.dir());
9933        match self
9934            .state
9935            .merge_repairs
9936            .last()
9937            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9938        {
9939            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9940            None => artifacts,
9941        }
9942    }
9943
9944    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
9945        if let Some(e2e) = self.state.head_e2e.take() {
9946            self.state
9947                .verification_history
9948                .push(crate::run::VerificationEvidence {
9949                    superseded_by: None,
9950                    archived_at: Some(Timestamp::now()),
9951                    fixes_spent: Some(self.state.gate_fixes.len()),
9952                    gate_head: None,
9953                    gate_ran: false,
9954                    gate: Vec::new(),
9955                    e2e: Some(e2e),
9956                });
9957        }
9958        self.state.reviews.clear();
9959        self.state.gate.clear();
9960        self.state.gate_ran = false;
9961        self.state.gate_head = None;
9962        self.state.gate_fixes.clear();
9963        self.state.gate_repair_e2e = None;
9964        self.state.pre_gate.clear();
9965        self.state.pre_gate_commit = None;
9966        self.state.contested_handoff = None;
9967        self.state.land_reruns = None;
9968        self.state.land_approval = None;
9969        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
9970        self.state.status = RunStatus::Reviewing;
9971        self.state.save()
9972    }
9973
9974    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
9975        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
9976        self.state.merge_repairs[index].failure = Some(why.clone());
9977        self.state.status = RunStatus::Blocked;
9978        self.state.event("repair", why);
9979        self.state.save()
9980    }
9981
9982    // -------------------------------------------------------------- helpers
9983
9984    /// Fetch or create a seat, keeping its conversation across nodes.
9985    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
9986        if let Some(existing) = self.state.seats.get(key)
9987            && existing.agent == agent
9988        {
9989            return existing.clone();
9990        }
9991        // A seat that changes agent mints its session id from the agent too,
9992        // like a handover: the old agent's uuid is already taken by the CLI.
9993        let fresh = if self.state.seats.contains_key(key) {
9994            handover_seat(key, agent, self.state.next_seat_seed())
9995        } else {
9996            SeatState::new(key, agent, self.state.seed)
9997        };
9998        self.state.seats.insert(key.to_owned(), fresh.clone());
9999        fresh
10000    }
10001
10002    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10003    /// seat to another roster agent, in which case that agent. Nodes that
10004    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10005    /// reconsideration) must keep talking to whoever answered it, not slip
10006    /// back to the agent that failed it.
10007    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10008        match self.state.seats.get(key) {
10009            Some(s) if s.agent != spec.id => {
10010                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10011            }
10012            _ => spec,
10013        }
10014    }
10015
10016    /// A candidate rendered for judging, with the leak policy applied.
10017    fn view(&self, c: &Candidate) -> CandidateView {
10018        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10019            .unwrap_or_default();
10020        let (patch, _) = blind::sanitize_patch(
10021            &format!("candidate {} patch", c.label),
10022            &raw,
10023            &self.state.config.blind,
10024        );
10025        CandidateView {
10026            label: c.label,
10027            branch: c.branch.clone(),
10028            summary: c.summary.clone(),
10029            stat: c.stat.clone(),
10030            patch,
10031        }
10032    }
10033
10034    /// The full candidate set as prompt text, for seats with no live session.
10035    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10036        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10037        prompt::judge(
10038            "(see above)",
10039            &views,
10040            self.roles.judges.len(),
10041            base_short,
10042            "en",
10043        )
10044    }
10045
10046    /// The final-vote prompt with everything a seat that has no session of its
10047    /// own needs: the candidates, the seat's own ranking and reasons, and the
10048    /// anonymised deliberation it took part in (only when there was one, so a
10049    /// handed-over seat never sees more than the seat it replaces did). The
10050    /// `Final vote` heading stays first.
10051    fn vote_prompt_full(
10052        &self,
10053        j: usize,
10054        labels: &[char],
10055        language: &str,
10056        candidates: &[Candidate],
10057        base_short: &str,
10058    ) -> String {
10059        let mut text = format!(
10060            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10061            prompt::final_vote(labels, language),
10062            self.state.instruction,
10063            self.candidate_block(candidates, base_short)
10064        );
10065        if let Some(own) = self
10066            .state
10067            .judgements
10068            .get(j)
10069            .filter(|r| !r.ranking.is_empty())
10070        {
10071            let reasons = own
10072                .reasons
10073                .iter()
10074                .map(|(k, v)| format!("- {k}: {v}"))
10075                .collect::<Vec<_>>()
10076                .join("\n");
10077            text.push_str(&format!(
10078                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10079                own.ranking.iter().collect::<String>(),
10080                if reasons.is_empty() {
10081                    ""
10082                } else {
10083                    ", because:\n"
10084                }
10085            ));
10086        }
10087        if !self.state.deliberation.is_empty() {
10088            text.push_str("\n# What was argued before this vote\n");
10089            for t in self.transcript(&[], j) {
10090                text.push_str(&format!(
10091                    "\n## {}{}\n\n{}\n",
10092                    t.who,
10093                    if t.is_self { " (you)" } else { "" },
10094                    t.body.trim()
10095                ));
10096            }
10097        }
10098        text
10099    }
10100
10101    /// Anonymised transcript for judge `self_idx`.
10102    ///
10103    /// The initial rankings are always the opening statements. Seeding them
10104    /// only when no turn had been taken yet meant every judge after the first
10105    /// argued against a single voice instead of against the actual split — the
10106    /// disagreement is the information, so it is always on the table.
10107    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10108        let mut turns = Vec::new();
10109        for j in &self.state.judgements {
10110            if j.ranking.is_empty() {
10111                continue;
10112            }
10113            let reasons = j
10114                .reasons
10115                .iter()
10116                .map(|(k, v)| format!("- {k}: {v}"))
10117                .collect::<Vec<_>>()
10118                .join("\n");
10119            turns.push(Turn {
10120                who: format!("Judge {} (opening ranking)", j.judge),
10121                is_self: j.judge == self_idx + 1,
10122                body: format!(
10123                    "Ranked {}{}{reasons}",
10124                    j.ranking.iter().collect::<String>(),
10125                    if reasons.is_empty() {
10126                        ""
10127                    } else {
10128                        ", because:\n"
10129                    }
10130                ),
10131            });
10132        }
10133        for t in self
10134            .state
10135            .deliberation
10136            .iter()
10137            .flat_map(|r| r.turns.iter())
10138            .chain(current)
10139        {
10140            turns.push(Turn {
10141                who: format!("Judge {}", t.judge),
10142                is_self: t.judge == self_idx + 1,
10143                body: t.body.clone(),
10144            });
10145        }
10146        turns
10147    }
10148}
10149
10150/// Does this seat still hold the context a follow-up prompt would rely on?
10151fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10152    agent::has_session(spec.kind, seat, sessions)
10153}
10154
10155/// The next entry in `roster` after `start`, never wrapping back to the
10156/// front, whose id is not in `tried` yet.
10157///
10158/// Starts one past `start` rather than at the front of `roster`: `start` is
10159/// the seat's own original position, and a seat whose candidate slot already
10160/// sits on the roster's second entry must fall through to the third next, not
10161/// restart at the first — which is very likely a different candidate's own
10162/// agent already. Never wraps back past `start`, for the same reason: an
10163/// entry earlier in the roster than the seat's own position is almost
10164/// certainly some *other* candidate slot's own agent, and once the tail of
10165/// the roster is exhausted there are no more untried agents for *this* seat
10166/// to fall through to — the caller's fallback chain ends there, exactly as
10167/// "no further untried agents remain in the list for that seat" asks for.
10168///
10169/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10170/// the same id twice (an operator's `roles.implementers` typo, or a
10171/// `[[agents]]` list reused across roles) must not let
10172/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10173/// over `roster` either finds an untried id or runs out, so this always
10174/// terminates regardless of duplicates.
10175fn next_untried_in_roster<'a>(
10176    roster: &'a [AgentSpec],
10177    start: usize,
10178    tried: &BTreeSet<String>,
10179) -> Option<&'a AgentSpec> {
10180    roster
10181        .get(start + 1..)?
10182        .iter()
10183        .find(|s| !tried.contains(&s.id))
10184}
10185
10186/// The successor for a seat, shared by all four seat kinds (implement, judge,
10187/// review, advise). `others` holds the ids that *currently* occupy the other
10188/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10189/// sees them), so roster entries beyond the seat count act as spares: a failed
10190/// seat goes to an agent no other seat holds whenever the roster permits.
10191///
10192/// `carried` is `Some` only for the review loop's carried failure history.
10193/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10194/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10195/// whole roster: untried and not another seat's occupant; (3) the plain walk
10196/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10197/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10198/// a better outcome than an empty seat, and it keeps the answer to "is there
10199/// a successor at all" exactly what it was before occupants were considered,
10200/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10201/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10202/// carried failure is a cheaper bet than doubling an agent on the panel, and
10203/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10204/// the same round are handled one at a time, so a seat later in the batch
10205/// sees an earlier one's new occupant but not yet its own freed agent.
10206fn pick_successor<'a>(
10207    roster: &'a [AgentSpec],
10208    start: usize,
10209    tried: &BTreeSet<String>,
10210    carried: Option<&BTreeSet<String>>,
10211    others: &BTreeSet<String>,
10212) -> Option<&'a AgentSpec> {
10213    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10214    roster
10215        .get(start + 1..)
10216        .and_then(|tail| {
10217            tail.iter()
10218                .filter(free)
10219                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10220        })
10221        .or_else(|| {
10222            carried
10223                .is_some()
10224                .then(|| roster.iter().find(free))
10225                .flatten()
10226        })
10227        .or_else(|| match carried {
10228            Some(c) => next_for_seat(roster, start, tried, c),
10229            None => next_untried_in_roster(roster, start, tried),
10230        })
10231}
10232
10233/// The next agent for a seat that carries its failure history across rounds
10234/// (the review loop). `round_tried` is this round's own bound and starts
10235/// empty every round; `carried_failed` only decides priority.
10236///
10237/// First: an id walking forward from `start`, never wrapping, that is neither
10238/// tried this round nor failed in an earlier one. Only when that is exhausted
10239/// does it rescue: the first roster id (in roster order, so this one *does*
10240/// look before `start`) not yet tried this round, which is by then a carried
10241/// failure. Each id is rescued at most once per round, so it cannot loop.
10242fn next_for_seat<'a>(
10243    roster: &'a [AgentSpec],
10244    start: usize,
10245    round_tried: &BTreeSet<String>,
10246    carried_failed: &BTreeSet<String>,
10247) -> Option<&'a AgentSpec> {
10248    roster
10249        .get(start + 1..)?
10250        .iter()
10251        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10252        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10253}
10254
10255/// Where a reviewer seat starts a round: the agent that last answered it when
10256/// it is still on the roster and not marked failed, else the spec's own agent
10257/// unless it failed, else the next roster agent that has not failed, else the
10258/// spec's own agent again (the whole roster failed). Ids no longer on the
10259/// roster are ignored. An empty roster has no handover, so the spec stands.
10260fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10261    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10262        return spec;
10263    };
10264    let ok = |id: &str| !h.failed.contains(id);
10265    if let Some(last) = h.last_ok.as_deref()
10266        && ok(last)
10267        && let Some(s) = roster.iter().find(|s| s.id == last)
10268    {
10269        return s.clone();
10270    }
10271    if ok(&spec.id) {
10272        return spec;
10273    }
10274    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10275    roster
10276        .get(from + 1..)
10277        .into_iter()
10278        .flatten()
10279        .chain(roster.iter())
10280        .find(|s| ok(&s.id))
10281        .cloned()
10282        .unwrap_or(spec)
10283}
10284
10285/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10286/// seed so a CLI that mints its session id up front (`--session-id`) never
10287/// reuses the uuid the previous agent already opened under the same seat key.
10288pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10289    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10290}
10291
10292/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10293/// so the classifier below and the code that builds the message cannot drift.
10294const TIMED_OUT: &str = "timed out";
10295
10296impl FailClass {
10297    /// `None` for an answer; otherwise how the turn failed.
10298    fn of(out: &AgentOutcome) -> Option<Self> {
10299        match out {
10300            AgentOutcome::Ok(_) => None,
10301            AgentOutcome::Quota(_) => Some(Self::Quota),
10302            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10303            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10304            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10305            // Not a failure of the agent. Callers check for it first; `None`
10306            // here means "no class to hand over on".
10307            AgentOutcome::ResourceBlocked(_) => None,
10308        }
10309    }
10310
10311    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10312    fn stem_word(&self) -> &'static str {
10313        match self {
10314            Self::Quota => "quota",
10315            _ => "handover",
10316        }
10317    }
10318}
10319
10320/// The message's first line with its variable parts removed — digit runs and
10321/// path-like tokens — so "exit code 2" and "exit code 7" read
10322/// as one kind of failure.
10323fn failure_signature(msg: &str) -> String {
10324    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10325    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10326    // the historical machine key while the process diagnosis uses readable prose.
10327    if let Some(code) = line
10328        .strip_prefix("process exited with no usable output (exit code ")
10329        .and_then(|s| s.strip_suffix(')'))
10330        .and_then(|s| s.parse::<i32>().ok())
10331    {
10332        line = format!("exited with some({code}) and no usable output");
10333    } else if matches!(
10334        line.as_str(),
10335        "process exited with no usable output (no exit code reported)"
10336            | "process exited with no usable output (exit code unavailable)"
10337    ) {
10338        line = "exited with none and no usable output".to_owned();
10339    }
10340    let mut out = Vec::new();
10341    for word in line.split_whitespace() {
10342        if word.contains('/') || word.contains('\\') {
10343            out.push("<path>".to_owned());
10344            continue;
10345        }
10346        let mut w = String::new();
10347        let mut in_digits = false;
10348        for c in word.chars() {
10349            if c.is_ascii_digit() {
10350                if !in_digits {
10351                    w.push('#');
10352                }
10353                in_digits = true;
10354            } else {
10355                in_digits = false;
10356                w.push(c);
10357            }
10358        }
10359        out.push(w);
10360    }
10361    out.join(" ").chars().take(120).collect()
10362}
10363
10364/// Whether a seat that just failed with `cur` may go to the next roster agent.
10365/// A quota or a timeout always may. Any other failure may not when the agent
10366/// before it failed the same way: an error the prompt causes would otherwise
10367/// walk the whole roster. `prev` is the class of the immediately preceding
10368/// agent's failure, so a quota or timeout in between breaks the run of
10369/// identical failures by itself.
10370fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10371    match cur {
10372        FailClass::Quota | FailClass::Timeout => true,
10373        FailClass::Other(_) => prev != Some(cur),
10374    }
10375}
10376
10377/// A short human reason for a failed outcome, for the handover record.
10378fn fail_reason(out: &AgentOutcome) -> String {
10379    match out {
10380        AgentOutcome::Ok(_) => String::new(),
10381        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10382        AgentOutcome::Dropped(o) => format!(
10383            "the CLI dropped the stream ({})",
10384            o.dropped
10385                .as_ref()
10386                .map(|d| d.why.as_str())
10387                .unwrap_or("it ended without delivering its answer")
10388        ),
10389        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10390            e.lines().next().unwrap_or("").chars().take(160).collect()
10391        }
10392    }
10393}
10394
10395/// Note one handover in the run: the structured record and, in the timeline,
10396/// the sentence a person reads. A quota keeps the wording it always had.
10397pub(crate) fn record_handover(
10398    state: &mut RunState,
10399    node: &str,
10400    seat: &str,
10401    from: &str,
10402    to: &str,
10403    class: &FailClass,
10404    reason: &str,
10405) {
10406    let message = if *class == FailClass::Quota {
10407        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10408    } else {
10409        format!("{seat}: handed over {from} -> {to} ({reason})")
10410    };
10411    state.event(node, message);
10412    state.handovers.push(Handover {
10413        at: Timestamp::now(),
10414        node: node.to_owned(),
10415        seat: seat.to_owned(),
10416        from: from.to_owned(),
10417        to: to.to_owned(),
10418        reason: reason.to_owned(),
10419    });
10420}
10421
10422/// Did this reply report running a command whose own CLI never confirmed an
10423/// exit status?
10424///
10425/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10426/// command *finished* (see that type's own doc), so this can only be `true`
10427/// for a command whose completion event carried no readable exit code — not
10428/// for one that simply is not mentioned at all. That is the one signal this
10429/// crate can read, from the same record `state.jobs` renders, about a reply
10430/// standing next to work its own CLI cannot vouch for finishing; it is
10431/// deliberately not a check on the exit code's *value* (a fixer legitimately
10432/// runs a command that fails mid-iteration before it succeeds) and not a
10433/// guess at a command still running in the background (which emits no event
10434/// at all, and so leaves no evidence here to find).
10435fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10436    commands.iter().any(|c| c.exit_code.is_none())
10437}
10438
10439/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10440/// trusted as a verified no-op — the adoption guard's own text-level half.
10441///
10442/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10443/// timed out): a marker only earns the benefit of the doubt from a turn the
10444/// CLI itself vouches for finishing properly, the same house style
10445/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10446/// *fix* report to for `commands`. A candidate that timed out, exited
10447/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10448/// is, whatever prose it wrote — this returns `None` before it ever looks at
10449/// `text`. The remaining guards (the tree really is empty, the evidence is
10450/// non-empty) are the caller's: this only reads what the reply *claimed*.
10451fn verified_noop_claim(
10452    usable: bool,
10453    commands: &[agent::CommandEvidence],
10454    text: &str,
10455) -> Option<String> {
10456    (usable && !has_unconfirmed_command(commands))
10457        .then(|| verdict::verified_noop(text))
10458        .flatten()
10459}
10460
10461fn short(commit: &str) -> String {
10462    commit.chars().take(7).collect()
10463}
10464
10465fn make_executable(path: &Path) -> Result<()> {
10466    #[cfg(unix)]
10467    {
10468        use std::os::unix::fs::PermissionsExt as _;
10469        let mut perms = std::fs::metadata(path)?.permissions();
10470        perms.set_mode(0o755);
10471        std::fs::set_permissions(path, perms)?;
10472    }
10473    #[cfg(not(unix))]
10474    {
10475        let _ = path;
10476    }
10477    Ok(())
10478}
10479
10480/// What every seat in one batch shares: where the answers are attributed, the
10481/// prompt overlay they inherit, and the build cache they are told to use.
10482///
10483/// A struct rather than four more parameters: `wave` also needs the run's
10484/// state (to record who is answering right now) and the attempt number, and
10485/// eight positional arguments is both unreadable and a clippy error.
10486struct WaveCtx<'a> {
10487    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10488    /// paid for it.
10489    run: &'a str,
10490    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10491    node: &'a str,
10492    prompts: &'a Prompts,
10493    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10494    cache: Option<&'a Path>,
10495    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10496    /// `JobRecord::round`. `None` for every other node.
10497    round: Option<usize>,
10498    /// Carry each seat's failed-agent history across waves (the review loop
10499    /// only): start-of-round priority and handover choice read
10500    /// [`RunState::seat_history`], and every answer or failure writes it.
10501    carry_seats: bool,
10502    /// The run's park / interrupt requests, so a wait for a shared resource
10503    /// can be abandoned the moment the operator asks to park.
10504    pauses: Option<(&'a Pause, &'a Pause)>,
10505}
10506
10507/// Shared scheduling policy for operations and spawned seat invocations.
10508fn repository_wait(
10509    state: &RunState,
10510    pauses: Option<(&Pause, &Pause)>,
10511) -> crate::repo_lock::WaitPolicy {
10512    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10513    let inherited = crate::repo_lock::WaitPolicy::current();
10514    crate::repo_lock::WaitPolicy::new(
10515        Duration::from_secs(state.config.graph.timeout_resource_wait),
10516        move || {
10517            inherited.cancelled()
10518                || pauses
10519                    .as_ref()
10520                    .is_some_and(|(a, b)| a.parked() || b.parked())
10521        },
10522    )
10523}
10524
10525fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10526    if error.is::<crate::repo_lock::ResourceBlocked>() {
10527        state.resource_block = Some(crate::run::ResourceBlock {
10528            cause: crate::run::ResourceBlockCause::Other,
10529            node: "repository".into(),
10530            kind: if error.to_string().contains("disk pressure") {
10531                "disk pressure"
10532            } else if error.to_string().contains("disk admission") {
10533                "disk admission"
10534            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10535                "incremental cache admission"
10536            } else {
10537                "repository coordination"
10538            }
10539            .into(),
10540            detail: error.to_string(),
10541            at: Timestamp::now(),
10542        });
10543        state.event("resource", error.to_string());
10544    }
10545}
10546
10547async fn coordinate_operation(
10548    state: &mut RunState,
10549    cwd: &Path,
10550    wait: &crate::repo_lock::WaitPolicy,
10551) -> Result<Option<crate::repo_lock::SeatLease>> {
10552    let result =
10553        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10554            .await;
10555    if let Err(e) = &result {
10556        record_repository_block(state, e);
10557    }
10558    result
10559}
10560
10561/// Run one job, honouring the parallelism budget.
10562async fn run_one(
10563    job: SeatJob,
10564    sem: Arc<Semaphore>,
10565    ctx: &WaveCtx<'_>,
10566    state: &mut RunState,
10567    attempt: usize,
10568) -> (SeatState, AgentOutcome) {
10569    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10570        .await
10571        .pop()
10572        .expect("one job in, one result out");
10573    (seat, out)
10574}
10575
10576/// Run every job concurrently, capped by the semaphore, preserving order.
10577///
10578/// Every seat in the batch is recorded into [`RunState::active`] before the
10579/// wave starts and cleared as each answer lands, so the run's own record says
10580/// who is still being waited on rather than only who finished.
10581async fn wave(
10582    jobs: Vec<SeatJob>,
10583    sem: Arc<Semaphore>,
10584    ctx: &WaveCtx<'_>,
10585    state: &mut RunState,
10586    attempt: usize,
10587) -> Vec<(usize, SeatState, AgentOutcome)> {
10588    let WaveCtx {
10589        run,
10590        node,
10591        prompts,
10592        cache,
10593        round,
10594        carry_seats: _,
10595        pauses,
10596    } = *ctx;
10597    for job in &jobs {
10598        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10599    }
10600    if let Err(e) = state.save() {
10601        // A failed persist of "who is answering right now" must not abort the
10602        // wave: the seats are already being asked, and the alternative is
10603        // losing the answers to save a status line nobody may even be
10604        // watching.
10605        tracing::warn!("could not persist in-progress seats: {e:#}");
10606    }
10607    // Hold the shared build cache's lease for the whole batch, not per job:
10608    // several candidates (an implement wave) or a fixer legitimately share
10609    // one cache concurrently within this run, and that stays untouched — a
10610    // single lease taken once for the whole wave and released once it is
10611    // done is what stops a *different* borrower (another run's own wave, its
10612    // e2e/gate, a human's `magi review`) from interleaving a build into the
10613    // same directory while this one is in flight. Best-effort, not
10614    // all-or-nothing: a wave that cannot get the lease within its own
10615    // longest job's budget still runs — an hour of paid implementer calls is
10616    // not thrown away over cache contention — but every write-allowed seat
10617    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10618    // below), the same fallback a read-only seat always gets, rather than
10619    // building into a directory this run was never granted. The identity
10620    // record is still invalidated below either way, so the next tracked
10621    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10622    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10623    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10624    // never borrowed from a job's agent budget (and never charged to it either,
10625    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10626    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10627    let cache_guard = if let Some(cache_dir) = cache {
10628        if jobs_had_a_writer {
10629            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10630            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10631            match acquire_cache_lease(
10632                state,
10633                cache_dir,
10634                &owner,
10635                resource_limit,
10636                node,
10637                node,
10638                &cancel,
10639            )
10640            .await
10641            {
10642                Ok(Some(g)) => Some(g),
10643                // Parking: leave without spawning anything. The run parks at
10644                // the next boundary and the seats are asked afresh on resume.
10645                Ok(None) => {
10646                    if let Some(block) = &mut state.resource_block {
10647                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10648                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10649                        };
10650                    }
10651                    return block_wave(
10652                        jobs,
10653                        state,
10654                        node,
10655                        attempt,
10656                        "parked while waiting for the shared build cache",
10657                    );
10658                }
10659                // An unreadable lease is not contention: keep the old
10660                // fallback (no shared cache for this wave's seats).
10661                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10662                Err(e) => {
10663                    // Writing seats may not build into a cache this run does
10664                    // not hold, and an agent that cannot be given one is not
10665                    // asked at all: the run stops resumably instead.
10666                    let why = format!("{e:#}");
10667                    state.event(
10668                        "resource",
10669                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10670                    );
10671                    return block_wave(jobs, state, node, attempt, &why);
10672                }
10673            }
10674        } else {
10675            None
10676        }
10677    } else {
10678        None
10679    };
10680    // Every job keeps its whole budget: the wait above was not the agent's.
10681    // Restart the seats' clocks so "started" means "being spawned now".
10682    state.seats_spawning(node, attempt);
10683    if let Err(e) = state.save() {
10684        tracing::warn!("could not persist the start of a wave: {e:#}");
10685    }
10686    // Queueing behind this wave's own seats for a parallelism slot is normal
10687    // (each of them is bounded by its own agent timeout); only a wait beyond
10688    // all of them plus the resource limit means the slots never came back.
10689    let slot_limit = resource_limit
10690        + jobs
10691            .iter()
10692            .map(|j| j.timeout)
10693            .max()
10694            .unwrap_or_default()
10695            .saturating_mul(jobs.len() as u32);
10696    let mut set = tokio::task::JoinSet::new();
10697    // Seats announce themselves here the moment they hold a parallelism
10698    // permit, i.e. just before their CLI is spawned.
10699    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10700    let overlay = prompts.overlay(node);
10701    let repository_wait = repository_wait(state, pauses);
10702    for (i, mut job) in jobs.into_iter().enumerate() {
10703        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10704        if cache.is_some() {
10705            job.prompt.push('\n');
10706            job.prompt
10707                .push_str(&prompt::build_cache_note(node, job.allow_write));
10708        }
10709        let sem = Arc::clone(&sem);
10710        let run = run.to_owned();
10711        let node = node.to_owned();
10712        // Only implementers were told about the task's attachments, so only
10713        // their seats get the directory widened for reading.
10714        let attachments = if node == "implement" {
10715            state.attachments.clone()
10716        } else {
10717            Vec::new()
10718        };
10719        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10720        // `prompt::build_cache_note`'s doc for why setting it anyway is
10721        // exactly how a sandboxed reviewer's write refusal got reported as a
10722        // defect in the patch, not a property of its own seat. And a
10723        // write-allowed one is handed it only when the lease above was
10724        // actually acquired: a wave that could not get it (`cache_guard` is
10725        // `None`, see its own comment) must not send seats to build into a
10726        // directory this run does not hold - that is the exact concurrent,
10727        // unmanaged-write race this module exists to prevent, not something
10728        // "proceeding anyway" is allowed to reintroduce.
10729        let cache = cache
10730            .filter(|_| job.allow_write && cache_guard.is_some())
10731            .map(Path::to_path_buf);
10732        let spawned_tx = spawned_tx.clone();
10733        let repository_wait = repository_wait.clone();
10734        let coordination = state.config.coordination.clone();
10735        let fetched_base = state.fetched_base.clone();
10736        set.spawn(async move {
10737            // A wait for a parallelism slot is scheduling, not the agent: it
10738            // is bounded by its own limit and ends as a resource block, never
10739            // as a timeout of the agent that was not started.
10740            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10741                Ok(p) => p,
10742                Err(_) => {
10743                    return (
10744                        i,
10745                        job.seat,
10746                        AgentOutcome::ResourceBlocked(format!(
10747                            "no parallel slot became free within {}s",
10748                            slot_limit.as_secs()
10749                        )),
10750                        false,
10751                    );
10752                }
10753            };
10754            spawned_tx.send(job.seat.key.clone()).ok();
10755            let mut seat = job.seat;
10756            let out = repository_wait
10757                .scope(agent::invoke(
10758                    &job.spec,
10759                    &mut seat,
10760                    &Invocation {
10761                        coordination: coordination.as_ref(),
10762                        fetched_base: fetched_base.as_ref(),
10763                        cwd: &job.cwd,
10764                        prompt: &job.prompt,
10765                        timeout: job.timeout,
10766                        allow_write: job.allow_write,
10767                        sessions: job.sessions,
10768                        artifacts: &job.artifacts,
10769                        stem: &job.stem,
10770                        run: &run,
10771                        node: &node,
10772                        cache_dir: cache.as_deref(),
10773                        attachments: &attachments,
10774                        writable: &[],
10775                    },
10776                ))
10777                .await;
10778            let interrupted = out
10779                .as_ref()
10780                .err()
10781                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10782            let out = match out {
10783                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10784                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10785                // Billed work the CLI failed to hand over is not an ordinary
10786                // failure, but its text is the CLI's raw error JSON, not an
10787                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10788                // read it as one by forgetting to check. `usable()` is always
10789                // false here (dropped implies an empty response), so this has
10790                // to be checked before the catch-all `Failed` below or the
10791                // one shape this exists for is lost with the rest.
10792                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10793                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10794                Ok(o) => AgentOutcome::Failed(format!(
10795                    "process exited with no usable output ({})",
10796                    crate::notices::exit_status(o.exit_code)
10797                )),
10798                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10799                    AgentOutcome::ResourceBlocked(e.to_string())
10800                }
10801                Err(e) => AgentOutcome::Failed(e.to_string()),
10802            };
10803            (i, seat, out, interrupted)
10804        });
10805    }
10806    drop(spawned_tx);
10807    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10808    loop {
10809        let joined = tokio::select! {
10810            Some(key) = spawned_rx.recv() => {
10811                // The clock of this seat starts now, not when it queued.
10812                state.seat_spawned(&key);
10813                if let Err(e) = state.save() {
10814                    tracing::warn!("could not persist a seat's start: {e:#}");
10815                }
10816                continue;
10817            }
10818            joined = set.join_next() => joined,
10819        };
10820        let Some(joined) = joined else {
10821            break;
10822        };
10823        let (i, seat, out, interrupted) = match joined {
10824            Ok(v) => v,
10825            // No seat to clear: a panicked task never reported which one it
10826            // was. The defensive sweep below this loop is what stops that
10827            // seat's `active` entry from surviving forever.
10828            Err(e) => {
10829                tracing::error!("agent task panicked: {e}");
10830                continue;
10831            }
10832        };
10833        state.seat_finished(&seat.key);
10834        if interrupted
10835            && node == "implement"
10836            && let Some(label) = state
10837                .candidates
10838                .iter()
10839                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10840                .map(|candidate| candidate.label)
10841        {
10842            state.candidate_preservation.insert(
10843                label,
10844                "interrupted implementation awaiting persistence".into(),
10845            );
10846            state.interrupted_implementations.insert(label);
10847        }
10848        if let AgentOutcome::ResourceBlocked(why) = &out
10849            && state.resource_block.is_none()
10850        {
10851            state.resource_block = Some(crate::run::ResourceBlock {
10852                cause: crate::run::ResourceBlockCause::Other,
10853                node: node.to_owned(),
10854                kind: if why.contains("disk pressure") {
10855                    "disk pressure"
10856                } else if why.contains("disk admission") {
10857                    "disk admission"
10858                } else if why.contains(crate::incremental::BLOCKED) {
10859                    "incremental cache admission"
10860                } else if why.contains("repository coordination") {
10861                    "repository coordination"
10862                } else {
10863                    "parallel slot"
10864                }
10865                .to_owned(),
10866                detail: why.clone(),
10867                at: Timestamp::now(),
10868            });
10869            let activity = if interrupted {
10870                "agent execution interrupted"
10871            } else {
10872                "no agent was started"
10873            };
10874            state.event("resource", format!("{node}: {activity} - {why}"));
10875        }
10876        record_jobs(state, node, round, &seat.key, &out);
10877        if let Err(e) = state.save() {
10878            tracing::warn!("could not persist a seat's completion: {e:#}");
10879        }
10880        if collected.len() <= i {
10881            collected.resize_with(i + 1, || None);
10882        }
10883        collected[i] = Some((i, seat, out));
10884    }
10885    // Belt-and-braces for the panic branch above: every seat this exact batch
10886    // started shares this `(node, attempt)` pair, and every seat that finished
10887    // normally already cleared itself, so anything left tagged with it here
10888    // can only be a panicked task's leftover. Cleared unconditionally rather
10889    // than left to read as still answering forever.
10890    if state
10891        .active
10892        .values()
10893        .any(|a| a.node == node && a.attempt == attempt)
10894    {
10895        state
10896            .active
10897            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10898        if let Err(e) = state.save() {
10899            tracing::warn!("could not persist the end of a wave: {e:#}");
10900        }
10901    }
10902    // Whether or not the lease above was actually held, several worktrees
10903    // may just have built into the cache with nothing here able to name one
10904    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10905    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10906    // clean it might not have strictly needed; trusting a stale match would
10907    // cost it a wrong answer.
10908    if let Some(cache_dir) = cache
10909        && jobs_had_a_writer
10910    {
10911        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10912    }
10913    if let Some(guard) = cache_guard {
10914        guard.release();
10915    }
10916    collected.into_iter().flatten().collect()
10917}
10918
10919/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10920/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10921/// this is data collection, not the fix-specific completion contract in
10922/// [`Runner::continue_fix_report`], and applies regardless of which node
10923/// asked.
10924///
10925/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10926/// evidence from; `Failed` does not, and correctly contributes nothing — a
10927/// timeout or crash is not itself evidence about a command the seat may have
10928/// started.
10929fn record_jobs(
10930    state: &mut RunState,
10931    node: &str,
10932    round: Option<usize>,
10933    seat: &str,
10934    out: &AgentOutcome,
10935) {
10936    let commands: &[agent::CommandEvidence] = match out {
10937        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10938        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10939    };
10940    let checked_at = Timestamp::now();
10941    for c in commands {
10942        state.jobs.push(JobRecord {
10943            node: node.to_owned(),
10944            round,
10945            seat: seat.to_owned(),
10946            id: c.id.clone(),
10947            description: c.description.clone(),
10948            checked_at,
10949            status: match c.exit_code {
10950                Some(0) => JobStatus::Completed,
10951                Some(_) => JobStatus::Failed,
10952                None => JobStatus::Unknown,
10953            },
10954            exit_code: c.exit_code,
10955            result_summary: c.result_summary.clone(),
10956            source: c.source.clone(),
10957        });
10958    }
10959}
10960
10961/// Is a review round clean, given how many reviewer seats answered against
10962/// how many the round expected?
10963///
10964/// A seat that never answered (timeout, crash, unparsable output) is not a
10965/// seat that read the patch and found nothing — treating it as such is
10966/// exactly the bug this function exists to close. Under the default `block`
10967/// policy a missing seat can never be clean; `warn` still requires the seats
10968/// that *did* answer to have found nothing blocking and verification to be
10969/// green.
10970///
10971/// `quota_missing` narrows that `block` default for exactly one cause of
10972/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
10973/// a session limit lifts by the very next round buys nothing — the seat is
10974/// asked again with the same quota — so once every missing seat is accounted
10975/// for by a quota loss (and at least one seat *did* answer, so a decision has
10976/// something to rest on) the round is decided on the panel that could answer,
10977/// same as `warn` would. A panel that lost every seat to quota is not
10978/// decided here: `answered == 0` falls through to the existing `block`
10979/// fallback so a fully collapsed panel still waits rather than landing on no
10980/// review at all.
10981fn round_is_clean(
10982    blocking: usize,
10983    e2e_ok: bool,
10984    answered: usize,
10985    expected: usize,
10986    quota_missing: usize,
10987    policy: IncompleteReviewPolicy,
10988) -> bool {
10989    if blocking != 0 || !e2e_ok {
10990        return false;
10991    }
10992    if answered == expected || policy == IncompleteReviewPolicy::Warn {
10993        return true;
10994    }
10995    answered > 0 && expected - answered <= quota_missing
10996}
10997
10998/// The review loop's own conclusion, derived entirely from its persisted
10999/// round records and the round budget that produced them — never from
11000/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11001/// recomputes the identical answer regardless of what an earlier node in the
11002/// same walk, or a previous walk, did to `status`.
11003///
11004/// `None` while more rounds remain to try, including when review never ran
11005/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11006/// gone clean, or the budget is spent, or the tree has stopped moving (see
11007/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11008///
11009/// - An incomplete panel that raised nothing is missing input, not a
11010///   verified tree — never a hand-off candidate, whatever verification said
11011///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11012/// - Otherwise, green e2e on the last round hands off (see
11013///   [`Runner::stop_reviewing`]); red e2e blocks.
11014///
11015/// A last round whose own verification is still `ResourceBlocked` — magi
11016/// itself never got a command to run, not evidence the patch is broken —
11017/// is neither: this returns `None` for it too, the same as "more rounds
11018/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11019/// handling of that shape) instead of this cheap recomputation guessing a
11020/// verdict a real attempt never produced.
11021pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11022    if max_rounds == 0
11023        || reviews.last().is_some_and(|r| {
11024            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11025        })
11026    {
11027        return Some(RunStatus::Gating);
11028    }
11029    let last = reviews.last()?;
11030    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11031    if reviews.len() < max_rounds && !stagnant {
11032        return None;
11033    }
11034    if last.incomplete() && last.blocking == 0 {
11035        return Some(RunStatus::Blocked);
11036    }
11037    if last.e2e_status() == E2eStatus::ResourceBlocked
11038        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11039    {
11040        return None;
11041    }
11042    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11043        RunStatus::Gating
11044    } else {
11045        RunStatus::Blocked
11046    })
11047}
11048
11049/// How long a re-ask may take, given the budget the first attempt had.
11050///
11051/// A `nudged` retry is a request to restate an answer the seat has already
11052/// worked out: it carries no new work, so it does not deserve the original
11053/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11054/// 133 seconds while a third sat for over ten minutes on a resumed session
11055/// holding 410 KB of prior output - and because the retry had inherited the
11056/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11057/// judging round whose other seats were long finished.
11058///
11059/// A quarter of the budget, with a floor so that a deliberately short timeout
11060/// does not collapse to nothing. A retry that re-sends the whole prompt
11061/// (because the seat kept no context) is the original job again, and keeps the
11062/// original budget.
11063pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11064    if nudged {
11065        (full / 4).max(Duration::from_secs(120)).min(full)
11066    } else {
11067        full
11068    }
11069}
11070
11071/// Run a wave and parse each reply, re-asking the seats whose reply was
11072/// unusable.
11073///
11074/// The re-ask is a nudge rather than the whole prompt again when the seat still
11075/// holds its conversation, which is the difference between a cheap retry and
11076/// paying for the entire candidate set twice.
11077///
11078/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11079/// successor in `roster` is handed to it instead of being re-asked: the
11080/// handover is the retry. A seat with no successor left (a single-agent
11081/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11082/// the asks to one seat in one node number at most
11083/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11084/// is asked once (a dropped stream is nudged first), and only the last agent
11085/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11086/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11087/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11088/// other failure stops the chain when the previous agent failed the same way
11089/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11090/// [`has_context`] is false and the job's own full prompt and full budget are
11091/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11092/// intermediate ones are not losses) and is returned as a failure like any
11093/// other absent seat — the caller decides whether the panel still has a
11094/// quorum. An empty `roster` disables handover: failures are nudged as they
11095/// always were, and a quota is simply lost. A reply that fails to parse or
11096/// validate is the prompt's doing and is only ever nudged, never handed over.
11097///
11098/// The returned [`SeatState`] names the agent that answered (or tried last).
11099#[allow(clippy::too_many_arguments)]
11100async fn ask_json_wave<T>(
11101    jobs: Vec<SeatJob>,
11102    sem: Arc<Semaphore>,
11103    retries: usize,
11104    roster: &[AgentSpec],
11105    ctx: &WaveCtx<'_>,
11106    losses: &mut Vec<QuotaLoss>,
11107    state: &mut RunState,
11108    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11109) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11110where
11111    T: serde::de::DeserializeOwned + Send + 'static,
11112{
11113    ask_wave_with(
11114        jobs,
11115        sem,
11116        retries,
11117        roster,
11118        ctx,
11119        losses,
11120        state,
11121        &|text: &str| {
11122            let v = verdict::extract_json::<T>(text)?;
11123            validate(&v)?;
11124            Ok(v)
11125        },
11126    )
11127    .await
11128}
11129
11130/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11131/// a node whose answer is prose (deliberation) shares the same handover,
11132/// failure classification, quota bookkeeping and bounds instead of a copy.
11133///
11134/// A seat handed to another roster agent is sent the job's `handover` prompt
11135/// (when it has one) rather than `prompt`: the new agent has no session, so a
11136/// resume-style prompt would be incomplete. That holds for the handover ask
11137/// and for every nudge to that agent whose `has_context` is false.
11138#[allow(clippy::too_many_arguments)]
11139async fn ask_wave_with<T>(
11140    jobs: Vec<SeatJob>,
11141    sem: Arc<Semaphore>,
11142    retries: usize,
11143    roster: &[AgentSpec],
11144    ctx: &WaveCtx<'_>,
11145    losses: &mut Vec<QuotaLoss>,
11146    state: &mut RunState,
11147    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11148) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11149where
11150    T: Send + 'static,
11151{
11152    let n = jobs.len();
11153    let originals: Vec<SeatJob> = jobs;
11154    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11155    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11156    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11157    // answer, N once it has gone through N nudges. Read back once this
11158    // returns, so a caller building a history record (`ReviewRecord`) can
11159    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11160    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11161    // field's own doc.
11162    let mut nudges: Vec<usize> = vec![0; n];
11163    // The agent now occupying each seat, the ids it has already been through,
11164    // where in the roster the walk began, the class of the last failure, and
11165    // the stem word of a handover not yet asked (full prompt, full budget).
11166    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11167    let mut tried: Vec<BTreeSet<String>> = specs
11168        .iter()
11169        .map(|s| BTreeSet::from([s.id.clone()]))
11170        .collect();
11171    let starts: Vec<usize> = specs
11172        .iter()
11173        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11174        .collect();
11175    let carry = ctx.carry_seats && !roster.is_empty();
11176    // Carried across rounds: ids that failed the seat earlier, and how the
11177    // last failure went (so a repeat of it is not handed over again).
11178    let carried: Vec<BTreeSet<String>> = originals
11179        .iter()
11180        .map(|j| {
11181            state
11182                .seat_history
11183                .get(&j.seat.key)
11184                .filter(|_| carry)
11185                .map(|h| h.failed.clone())
11186                .unwrap_or_default()
11187        })
11188        .collect();
11189    let mut prev: Vec<Option<FailClass>> = originals
11190        .iter()
11191        .map(|j| {
11192            state
11193                .seat_history
11194                .get(&j.seat.key)
11195                .filter(|_| carry)
11196                .and_then(|h| h.last_fail.clone())
11197        })
11198        .collect();
11199    let next_agent =
11200        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11201            let others: BTreeSet<String> = specs
11202                .iter()
11203                .enumerate()
11204                .filter(|(j, _)| *j != i)
11205                .map(|(_, s)| s.id.clone())
11206                .collect();
11207            pick_successor(
11208                roster,
11209                starts[i],
11210                tried,
11211                carry.then(|| &carried[i]),
11212                &others,
11213            )
11214            .cloned()
11215        };
11216    let mut fresh: Vec<Option<String>> = vec![None; n];
11217    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11218    let mut pending: Vec<usize> = (0..n).collect();
11219
11220    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11221    // asks: an agent with a successor is asked once and handed over, and only
11222    // a seat with no successor spends `retries` nudges on the same agent. This
11223    // only guarantees the loop's own termination whatever those say.
11224    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11225    for round in 0..max_rounds {
11226        if pending.is_empty() {
11227            break;
11228        }
11229        let mut batch = Vec::with_capacity(pending.len());
11230        let mut renudged: Vec<&str> = Vec::new();
11231        for &i in &pending {
11232            let src = &originals[i];
11233            // A seat now held by another agent than the job named has no
11234            // session of its own: it gets the full-context prompt whenever
11235            // it is asked in full (the handover ask, a nudge it cannot
11236            // resume).
11237            let full: &str = match &src.handover {
11238                Some(h) if specs[i].id != src.spec.id => h,
11239                _ => &src.prompt,
11240            };
11241            // The prompt and the budget are one decision: a nudge restates
11242            // finished work, a re-sent prompt redoes it.
11243            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11244                (
11245                    full.to_owned(),
11246                    src.timeout,
11247                    format!("{}-{word}-{}", src.stem, specs[i].id),
11248                )
11249            } else if nudges[i] == 0 {
11250                (full.to_owned(), src.timeout, src.stem.clone())
11251            } else {
11252                renudged.push(src.seat.key.as_str());
11253                let why = done[i]
11254                    .as_ref()
11255                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11256                    .unwrap_or_else(|| "no parsable answer".to_owned());
11257                let nudge = prompt::nudge(&why);
11258                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11259                let prompt = if nudged {
11260                    nudge
11261                } else {
11262                    format!("{full}\n\n---\n\n{nudge}")
11263                };
11264                (
11265                    prompt,
11266                    retry_budget(src.timeout, nudged),
11267                    format!("{}-retry{}", src.stem, nudges[i]),
11268                )
11269            };
11270            batch.push(SeatJob {
11271                spec: specs[i].clone(),
11272                seat: seats[i].clone(),
11273                cwd: src.cwd.clone(),
11274                prompt,
11275                timeout,
11276                allow_write: src.allow_write,
11277                sessions: src.sessions,
11278                artifacts: src.artifacts.clone(),
11279                stem,
11280                handover: None,
11281            });
11282        }
11283
11284        if !renudged.is_empty() {
11285            state.event(
11286                ctx.node,
11287                format!("retry {round}: re-asking {}", renudged.join(", ")),
11288            );
11289        }
11290        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11291        let mut still = Vec::new();
11292        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11293            seats[i] = seat;
11294            // No agent was asked: nothing to hand over, nudge, remember or
11295            // charge. The caller sees `resource_block` and stops the node.
11296            if let AgentOutcome::ResourceBlocked(why) = &out {
11297                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11298                continue;
11299            }
11300            let class = FailClass::of(&out);
11301            // A dropped stream is nudged first (the conversation is still
11302            // there to pick up); only a seat whose nudges are spent hands over.
11303            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11304                && nudges[i] < retries
11305                && !roster.is_empty();
11306            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11307                let next = should_hand_over(prev[i].as_ref(), &cur)
11308                    .then(|| next_agent(i, &tried[i], &specs))
11309                    .flatten();
11310                if carry {
11311                    let h = state
11312                        .seat_history
11313                        .entry(originals[i].seat.key.clone())
11314                        .or_default();
11315                    h.failed.insert(specs[i].id.clone());
11316                    h.last_fail = Some(cur.clone());
11317                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11318                        h.last_ok = None;
11319                    }
11320                    // Saved before the next agent is asked, so a restart in
11321                    // between does not forget who failed.
11322                    if let Err(e) = state.save() {
11323                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11324                    }
11325                }
11326                if let Some(next) = next {
11327                    record_handover(
11328                        state,
11329                        ctx.node,
11330                        &originals[i].seat.key,
11331                        &specs[i].id,
11332                        &next.id,
11333                        &cur,
11334                        &fail_reason(&out),
11335                    );
11336                    tried[i].insert(next.id.clone());
11337                    prev[i] = Some(cur.clone());
11338                    seats[i] =
11339                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11340                    specs[i] = next;
11341                    fresh[i] = Some(cur.stem_word().to_owned());
11342                    nudges[i] = 0;
11343                    done[i] = Some(Err(anyhow::anyhow!(
11344                        "handed over after: {}",
11345                        fail_reason(&out)
11346                    )));
11347                    still.push(i);
11348                    continue;
11349                }
11350            }
11351            if carry
11352                && !nudge_first
11353                && let Some(cur) = class.clone()
11354            {
11355                // The chain ended here (no successor, or a repeated failure).
11356                let h = state
11357                    .seat_history
11358                    .entry(originals[i].seat.key.clone())
11359                    .or_default();
11360                h.failed.insert(specs[i].id.clone());
11361                h.last_fail = Some(cur);
11362            }
11363            let parsed = match out {
11364                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11365                AgentOutcome::Quota(o) => {
11366                    last_quota[i] = o.quota.clone();
11367                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11368                }
11369                // Not a parseable answer: the nudge loop re-asks it, which is
11370                // exactly what a dropped stream needs. Just don't hand its raw
11371                // error JSON to `extract_json`.
11372                AgentOutcome::Dropped(o) => {
11373                    let why = o
11374                        .dropped
11375                        .as_ref()
11376                        .map(|d| d.why.as_str())
11377                        .unwrap_or("the CLI ended the stream without delivering its answer");
11378                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11379                }
11380                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11381                    Err(anyhow::anyhow!(e))
11382                }
11383            };
11384            let quota = class == Some(FailClass::Quota);
11385            let failed = parsed.is_err();
11386            if !failed && matches!(ctx.node, "judge" | "vote") {
11387                // A validated answer restores this seat's quorum presence,
11388                // including on the normal path after an all-seat block.
11389                state
11390                    .quota
11391                    .retain(|loss| loss.seat != originals[i].seat.key);
11392            }
11393            done[i] = Some(parsed);
11394            if carry && !failed {
11395                let h = state
11396                    .seat_history
11397                    .entry(originals[i].seat.key.clone())
11398                    .or_default();
11399                h.failed.remove(&specs[i].id);
11400                h.last_ok = Some(specs[i].id.clone());
11401                h.last_fail = None;
11402            }
11403            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11404            // fail the same way; and never re-ask a seat that already parsed.
11405            // A failed agent that still has a successor is not re-asked
11406            // either: the handover was its remedy and has just been refused
11407            // (the chain stops on a repeated failure class). A seat with no
11408            // successor left (a single-agent roster, the roster's tail, or an
11409            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11410            let agent_failure = class.is_some()
11411                && !nudge_first
11412                && !roster.is_empty()
11413                && next_agent(i, &tried[i], &specs).is_some();
11414            if failed && !quota && !agent_failure && nudges[i] < retries {
11415                nudges[i] += 1;
11416                still.push(i);
11417            }
11418        }
11419        pending = still;
11420    }
11421
11422    // One loss per seat whose chain ended on a quota: the intermediate ones
11423    // were absorbed by a handover and are not losses.
11424    for (i, q) in last_quota.into_iter().enumerate() {
11425        if let Some(quota) = q {
11426            losses.push(QuotaLoss::from_report(
11427                originals[i].seat.key.clone(),
11428                ctx.node,
11429                Some(&quota),
11430            ));
11431        }
11432    }
11433
11434    // A stalled quorum already has its own recovery checkpoint. Refresh
11435    // its missing seats before deciding whether it can resume.
11436    if state.status != RunStatus::Stalled
11437        && ctx.node != "deliberate"
11438        && n > 0
11439        && losses.len() == n
11440        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11441    {
11442        block_machine_outage(state, ctx.node, losses);
11443        state.quota.append(losses);
11444    }
11445
11446    seats
11447        .into_iter()
11448        .zip(done)
11449        .zip(nudges)
11450        .map(|((seat, res), attempts)| {
11451            (
11452                seat,
11453                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11454                attempts,
11455            )
11456        })
11457        .collect()
11458}
11459
11460/// A node with no available agent has no task verdict. Preserve the node
11461/// checkpoint and record why a subsequent explicit resume is needed.
11462pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11463    let mut auth: Vec<_> = losses
11464        .iter()
11465        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11466        .map(|loss| loss.agent.clone())
11467        .collect();
11468    auth.sort();
11469    auth.dedup();
11470    let (kind, detail) = if auth.is_empty() {
11471        (
11472            "agent usage limit",
11473            "All seats reached their agent usage limits; resume after usage is available"
11474                .to_owned(),
11475        )
11476    } else {
11477        (
11478            "agent authentication",
11479            format!(
11480                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11481                auth.join(", ")
11482            ),
11483        )
11484    };
11485    state.resource_block = Some(crate::run::ResourceBlock {
11486        node: node.to_owned(),
11487        kind: kind.to_owned(),
11488        cause: crate::run::ResourceBlockCause::Other,
11489        detail: detail.clone(),
11490        at: Timestamp::now(),
11491    });
11492    state.event(node, detail);
11493}
11494
11495/// Did the wave end without asking any agent (see [`block_wave`])?
11496fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11497    results
11498        .iter()
11499        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11500}
11501
11502/// Settle a wave that never spawned anything: every job answers
11503/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11504/// else about the seat (session, history, quota) is touched.
11505fn block_wave(
11506    jobs: Vec<SeatJob>,
11507    state: &mut RunState,
11508    node: &str,
11509    attempt: usize,
11510    why: &str,
11511) -> Vec<(usize, SeatState, AgentOutcome)> {
11512    state.resource_wait = None;
11513    state
11514        .active
11515        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11516    if let Err(e) = state.save() {
11517        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11518    }
11519    jobs.into_iter()
11520        .enumerate()
11521        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11522        .collect()
11523}
11524
11525/// Acquire the shared build cache's lease, waiting out contention within
11526/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11527/// unbounded wait is never acceptable).
11528///
11529/// A first, non-blocking check happens before ever waiting; if it finds the
11530/// lease busy, that fact is logged as a `verify` event *and* flushed with
11531/// [`RunState::save`] immediately — not only once the wait finally succeeds
11532/// or gives up — so a `magi show` run by a different process while this one
11533/// is still waiting reads a `run.json` that says so, rather than whatever it
11534/// looked like before the wait started. The same applies to the terminal
11535/// failure: logged and saved before this returns `Err`, so a caller that
11536/// could not get the lease at all still leaves a legible record of why.
11537async fn acquire_cache_lease(
11538    state: &mut RunState,
11539    cache_dir: &Path,
11540    owner: &crate::cache::Owner,
11541    budget: Duration,
11542    node: &str,
11543    context: &str,
11544    cancel: &(dyn Fn() -> bool + Sync),
11545) -> Result<Option<crate::cache::Guard>> {
11546    let home = crate::run::home();
11547    let started = Instant::now();
11548    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11549        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11550        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11551        Err(e) => {
11552            state.event(
11553                "verify",
11554                format!("{context}: could not check the shared build cache: {e:#}"),
11555            );
11556            if let Err(e2) = state.save() {
11557                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11558            }
11559            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11560        }
11561    };
11562    state.event(
11563        "verify",
11564        format!(
11565            "{context}: waiting for the shared build cache at {} ({})",
11566            cache_dir.display(),
11567            busy.describe()
11568        ),
11569    );
11570    // Seats of this node are waiting, not running: say so, so the countdown a
11571    // reader sees only ever measures an agent that has been spawned.
11572    state.resource_wait = Some(crate::run::ResourceWait {
11573        node: context.to_owned(),
11574        kind: "shared build cache".to_owned(),
11575        holder: busy.describe(),
11576        since: Timestamp::now(),
11577        limit_secs: budget.as_secs(),
11578    });
11579    if let Err(e) = state.save() {
11580        tracing::warn!("could not persist a cache wait: {e:#}");
11581    }
11582    let remaining = budget.saturating_sub(started.elapsed());
11583    let waited = crate::cache::wait_for_unless(
11584        &home,
11585        cache_dir,
11586        owner,
11587        remaining,
11588        Duration::from_secs(5),
11589        cancel,
11590    )
11591    .await;
11592    state.resource_wait = None;
11593    let cause = match &waited {
11594        Ok(Some(_)) => None,
11595        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11596        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11597            Some(crate::run::ResourceBlockCause::Timeout {
11598                waited_secs: started.elapsed().as_secs(),
11599                limit_secs: budget.as_secs(),
11600            })
11601        }
11602        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11603    };
11604    if let Some(cause) = cause {
11605        let detail = match &waited {
11606            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11607            Err(error) => format!("{error:#}"),
11608        };
11609        state.resource_block = Some(crate::run::ResourceBlock {
11610            cause,
11611            node: node.to_owned(),
11612            kind: "shared build cache".into(),
11613            detail,
11614            at: Timestamp::now(),
11615        });
11616        state.event(
11617            "resource",
11618            state.resource_block.as_ref().unwrap().description(),
11619        );
11620    }
11621    if let Err(error) = state.save() {
11622        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11623    }
11624    waited
11625}
11626
11627/// The lease state could not be read at all (unreadable lease file, a
11628/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11629/// known to be contended, so a wave degrades to running without a shared cache
11630/// instead of stopping on a hold that [`crate::triage`] could never release.
11631#[derive(Debug)]
11632struct CacheCheckError(String);
11633
11634impl std::fmt::Display for CacheCheckError {
11635    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11636        f.write_str(&self.0)
11637    }
11638}
11639
11640impl std::error::Error for CacheCheckError {}
11641
11642/// Keep a specific acquisition outcome when the caller adds command evidence.
11643fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11644    state
11645        .resource_block
11646        .get_or_insert_with(|| crate::run::ResourceBlock {
11647            cause: crate::run::ResourceBlockCause::Other,
11648            node: node.to_owned(),
11649            kind: "shared build cache".into(),
11650            detail: format!("{error:#}"),
11651            at: Timestamp::now(),
11652        });
11653    if let Err(error) = state.save() {
11654        tracing::warn!("could not persist cache resource block: {error:#}");
11655    }
11656}
11657
11658/// Run `body` — a verify command batch — while holding the shared build
11659/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11660/// never interleave with another borrower's build against the same
11661/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11662/// timeout, or a human's own `magi review`. See the `cache` module doc for
11663/// why this matters more than Cargo's own per-target locking covers — two
11664/// *different* worktrees building the same package name/version into one
11665/// cache directory is a staleness bug, not a lock contention one.
11666///
11667/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11668/// agent waves do. Repository scheduling also has its own bounded,
11669/// cancellable resource-wait policy. Contention must not consume the
11670/// verification budget: `body` receives the full `budget` after both leases
11671/// are acquired and the cache is fresh. Without a shared cache, only
11672/// repository coordination is acquired before handing over the full budget.
11673///
11674/// A lease that cannot be acquired within the resource wait is reported as a single
11675/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11676/// verification — the same shape a spawn failure already takes in
11677/// [`run_commands`], so a caller need not special-case it.
11678#[allow(clippy::too_many_arguments)]
11679async fn with_cache_lease<'s, F, Fut>(
11680    state: &'s mut RunState,
11681    cache_dir: Option<&Path>,
11682    node: &str,
11683    seat: &str,
11684    worktree: &Path,
11685    head: &str,
11686    budget: Duration,
11687    context: &str,
11688    wait: &crate::repo_lock::WaitPolicy,
11689    body: F,
11690) -> Result<(Vec<CommandOutcome>, bool)>
11691where
11692    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11693    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11694{
11695    // Match editing waves: cache first, repository second. In particular,
11696    // pre_gate must never hold the repository while waiting for a wave's cache.
11697    let started = Instant::now();
11698    let home = crate::run::home();
11699    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11700    let guard = if let Some(cache_dir) = cache_dir {
11701        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11702        match acquire_cache_lease(
11703            state,
11704            cache_dir,
11705            &owner,
11706            resource_limit,
11707            node,
11708            context,
11709            &|| wait.cancelled(),
11710        )
11711        .await
11712        .and_then(|g| {
11713            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11714        }) {
11715            Ok(g) => Some(g),
11716            Err(e) => {
11717                record_cache_error(state, node, &e);
11718                return Ok((
11719                    vec![CommandOutcome {
11720                        command: "(waiting for the shared build cache)".into(),
11721                        code: None,
11722                        output_tail: e.to_string(),
11723                        duration_ms: started.elapsed().as_millis() as u64,
11724                        resource_blocked: true,
11725                    }],
11726                    false,
11727                ));
11728            }
11729        }
11730    } else {
11731        None
11732    };
11733    let _coordination = coordinate_operation(state, worktree, wait).await?;
11734    if let Some(cache_dir) = cache_dir {
11735        let identity = crate::cache::Identity::new(worktree, head);
11736        if let Err(e) = crate::cache::ensure_fresh(
11737            &home,
11738            cache_dir,
11739            &identity,
11740            &state.config.verify.cargo_manifests(),
11741        ) {
11742            record_cache_error(state, node, &e);
11743            state.event(
11744                "verify",
11745                format!(
11746                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11747                    worktree.display(),
11748                    short(head)
11749                ),
11750            );
11751            if let Some(guard) = guard {
11752                guard.release();
11753            }
11754            return Ok((
11755                vec![CommandOutcome {
11756                    command: "(confirming the shared build cache is fresh)".into(),
11757                    code: None,
11758                    output_tail: e.to_string(),
11759                    duration_ms: started.elapsed().as_millis() as u64,
11760                    resource_blocked: true,
11761                }],
11762                false,
11763            ));
11764        }
11765    }
11766    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11767    if !timed_out_pids.is_empty() {
11768        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11769    }
11770    // Drop repository coordination before releasing the cache, reversing the
11771    // acquisition order. A cancelled/error future also drops both guards.
11772    drop(_coordination);
11773    if let Some(guard) = guard {
11774        guard.release();
11775    }
11776    Ok((outcomes, retried))
11777}
11778
11779/// Poll `pids` — commands [`run_commands`] reports as still running when its
11780/// own timeout elapsed — until every one is confirmed gone, or
11781/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11782///
11783/// Real confirmation where confirmation is possible, not a substitute for
11784/// full process-tree observation: a grandchild the timed-out process spawned
11785/// and that survives independently of it is invisible to a pid check the
11786/// same way it always was, and continuing to observe and collect *that*
11787/// stays a different piece of work with its own owner. This only narrows a
11788/// fixed blind wait into an actual check of the pids this process does know
11789/// about.
11790async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11791    wait_for_pids_with(
11792        pids,
11793        crate::proc::pid_alive,
11794        LEASE_RELEASE_POLL,
11795        LEASE_RELEASE_MAX_WAIT,
11796    )
11797    .await;
11798}
11799
11800/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11801/// interval and ceiling supplied by the caller, so the polling *logic* -
11802/// returns as soon as every pid reports dead, gives up at the ceiling
11803/// otherwise - is testable on millisecond durations without asking the real
11804/// OS about a pid at all.
11805async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11806    pids: &[u32],
11807    alive: F,
11808    poll: Duration,
11809    max_wait: Duration,
11810) {
11811    let deadline = Instant::now() + max_wait;
11812    loop {
11813        if pids.iter().all(|&pid| !alive(pid)) {
11814            return;
11815        }
11816        if Instant::now() >= deadline {
11817            return;
11818        }
11819        tokio::time::sleep(poll).await;
11820    }
11821}
11822
11823/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11824/// admission that it could not even get a verify command to run, as opposed
11825/// to evidence the command actually produced? A caller that would otherwise
11826/// read a resource-blocked outcome as a red command must check this first:
11827/// see [`Runner::gate`], which retries rather than records `Blocked` when
11828/// this is true.
11829fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11830    outcomes.iter().any(|o| o.resource_blocked)
11831}
11832
11833/// What [`Runner::gate_fix_round`] decided.
11834pub(crate) enum GateFix {
11835    /// The tree changed and `verify.e2e` is still green: run the gate again.
11836    Retry,
11837    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11838    /// last failure stands and the run ends blocked.
11839    Stop,
11840    /// `verify.e2e` could not run after the fix (magi's own contention):
11841    /// decide nothing now, a later reentry retries.
11842    Defer,
11843}
11844
11845/// Is every red command in `outcomes` an ordinary failure the code could
11846/// explain: it ran, exited non-zero, and said something?
11847///
11848/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11849/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11850/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11851/// what the command is stays the gate's business.
11852pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11853    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11854    red.peek().is_some()
11855        && red.all(|o| {
11856            !o.resource_blocked
11857                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11858                && !o.output_tail.trim().is_empty()
11859        })
11860}
11861
11862/// Describe one verify command's outcome for the event log, distinguishing a
11863/// build/link failure — the toolchain never produced a binary to run — from
11864/// an actual test failure, since only the latter is a verdict on the patch.
11865fn e2e_outcome_label(o: &CommandOutcome) -> String {
11866    if o.ok() {
11867        return "pass".to_owned();
11868    }
11869    let reason = if o.build_failed() {
11870        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11871    } else {
11872        format!("FAIL ({:?})", o.code)
11873    };
11874    format!(
11875        "{reason}\n{}",
11876        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11877    )
11878}
11879
11880/// Run `verify.e2e`, retrying once if the first attempt could not build or
11881/// link — a build/link failure is frequently a race against a shared
11882/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11883/// `verify` event per command, tagged with `context` (normally `"round N"`)
11884/// so the two call sites that need this — the ordinary per-round leg in
11885/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11886/// it will ever call a round green — read identically in the event log.
11887async fn run_e2e_with_retry(
11888    state: &mut RunState,
11889    shell: &[String],
11890    commands: &[String],
11891    worktree: &Path,
11892    timeout: Duration,
11893    context: &str,
11894) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11895    let (mut e2e, mut timed_out_pids) = run_commands(
11896        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11897    )
11898    .await;
11899    for o in &e2e {
11900        state.event(
11901            "verify",
11902            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11903        );
11904    }
11905    // A build/link failure is not a verdict on the patch — it is frequently a
11906    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11907    // one retry before letting a red like that decide the round.
11908    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11909    if verify_retried {
11910        state.event(
11911            "verify",
11912            format!(
11913                "{context}: verify could not build/link, not a test result — retrying once \
11914                 before concluding"
11915            ),
11916        );
11917        let retried = run_commands(
11918            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11919        )
11920        .await;
11921        e2e = retried.0;
11922        // Both attempts' timeouts matter, not just the last one: the first
11923        // attempt's descendants may still be alive alongside the retry's.
11924        timed_out_pids.extend(retried.1);
11925        for o in &e2e {
11926            state.event(
11927                "verify",
11928                format!(
11929                    "{context}: retry `{}` -> {}",
11930                    o.command,
11931                    e2e_outcome_label(o)
11932                ),
11933            );
11934        }
11935    }
11936    (e2e, verify_retried, timed_out_pids)
11937}
11938
11939/// Run configured shell commands in `cwd`, in order. The second element is
11940/// the pid of every command that hit `timeout` and was still running when
11941/// this stopped waiting on it (best-effort: `None` when the platform did not
11942/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
11943/// that releases a shared resource afterward needs to know.
11944///
11945/// Records `task` into [`RunState::active`] at every command boundary
11946/// (`RunState::task_command`) and clears it once the whole list has run
11947/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
11948/// for minutes with no seat and no output of its own to show for it (see
11949/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
11950/// apart from "not yet run" without this), and this is the only place that
11951/// knows which command is running right now and how many are left. Three
11952/// saves per command — start, not per second — matching the same "only at a
11953/// boundary" rule [`wave`] already follows for seats.
11954#[allow(clippy::too_many_arguments)]
11955async fn run_commands(
11956    state: &mut RunState,
11957    node: &str,
11958    task: &str,
11959    attempt: usize,
11960    shell: &[String],
11961    commands: &[String],
11962    cwd: &Path,
11963    timeout: Duration,
11964) -> (Vec<CommandOutcome>, Vec<u32>) {
11965    if commands.is_empty() {
11966        // Nothing to mark as running and nothing to clear — an empty list
11967        // means "not configured", and touching `active` (or the disk) over
11968        // that would be a write for every round of a repo with no
11969        // `verify.e2e` / `verify.gate` commands at all.
11970        return (Vec::new(), Vec::new());
11971    }
11972    let mut out = Vec::new();
11973    let mut timed_out_pids = Vec::new();
11974    let total = commands.len();
11975    for (idx, command) in commands.iter().enumerate() {
11976        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
11977        if let Err(e) = state.save() {
11978            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
11979        }
11980        let started = Instant::now();
11981        let mut cmd = tokio::process::Command::new(&shell[0]);
11982        cmd.quiet();
11983        // Verify the checkout as a standalone caller. Worker ownership and
11984        // pinned executable selection belong to this driver, not to tests or
11985        // tools it runs. Preserve home/seat context and Running's containment.
11986        for key in crate::worker::INVOCATION_ENV {
11987            cmd.env_remove(key);
11988        }
11989        cmd.args(&shell[1..])
11990            .arg(command)
11991            .current_dir(cwd)
11992            .stdin(std::process::Stdio::null())
11993            .stdout(std::process::Stdio::piped())
11994            .stderr(std::process::Stdio::piped())
11995            .kill_on_drop(true);
11996        let mut outputs = Vec::new();
11997        let protected = (|| -> Result<()> {
11998            crate::incremental::configure(&mut cmd, cwd)?;
11999            let mut paths = vec![cwd.to_owned()];
12000            if let Some(cache) = state.config.cache_dir() {
12001                paths.push(cache);
12002            }
12003            outputs.extend(crate::retention::OutputLease::admit(
12004                &crate::run::home(),
12005                &paths,
12006                state.config.disk.min_free_bytes,
12007            )?);
12008            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12009                outputs.push(guard);
12010            }
12011            if let Some(cache) = state.config.cache_dir() {
12012                outputs.push(crate::retention::OutputLease::cache(
12013                    &crate::run::home(),
12014                    &cache,
12015                    false,
12016                )?);
12017            }
12018            Ok(())
12019        })();
12020        if let Err(e) = protected {
12021            let detail = format!("execution admission unavailable: {e:#}");
12022            state.resource_block = Some(crate::run::ResourceBlock {
12023                cause: crate::run::ResourceBlockCause::Other,
12024                node: node.to_owned(),
12025                kind: if detail.contains("disk admission") {
12026                    "disk admission"
12027                } else {
12028                    "output ownership"
12029                }
12030                .to_owned(),
12031                detail: detail.clone(),
12032                at: Timestamp::now(),
12033            });
12034            out.push(CommandOutcome {
12035                command: command.clone(),
12036                code: None,
12037                output_tail: detail,
12038                duration_ms: 0,
12039                resource_blocked: true,
12040            });
12041            break;
12042        }
12043        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12044        let (code, body) = match spawned {
12045            Ok(mut running) => {
12046                let pid = running.child().id();
12047                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12048                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12049                let mut paths = vec![cwd.to_owned()];
12050                if let Some(cache) = state.config.cache_dir() {
12051                    paths.push(cache);
12052                }
12053                let mut pressure = None;
12054                let status = tokio::select! {
12055                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12056                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12057                        pressure = Some(reason); None
12058                    }
12059                };
12060                let (code, failure) = match status {
12061                    Some(Ok(Ok(status))) => (
12062                        status.code(),
12063                        status
12064                            .code()
12065                            .is_none()
12066                            .then(|| format!("command terminated: {status}")),
12067                    ),
12068                    Some(Ok(Err(error))) => {
12069                        (None, Some(format!("failed waiting for command: {error}")))
12070                    }
12071                    Some(Err(_)) => {
12072                        // Leased cleanup confirms process-tree death itself.
12073                        // Unleased commands still need the caller's PID check
12074                        // after kill-on-drop, before releasing cache ownership.
12075                        if !running.coordinated()
12076                            && let Some(pid) = pid
12077                        {
12078                            timed_out_pids.push(pid);
12079                        }
12080                        (
12081                            None,
12082                            Some(format!("timed out after {}s", timeout.as_secs())),
12083                        )
12084                    }
12085                    None => (None, None),
12086                };
12087                running.finish().await;
12088                if let Some(reason) = pressure {
12089                    state.resource_block = Some(crate::run::ResourceBlock {
12090                        cause: crate::run::ResourceBlockCause::Other,
12091                        node: node.to_owned(),
12092                        kind: "disk pressure".to_owned(),
12093                        detail: reason.clone(),
12094                        at: Timestamp::now(),
12095                    });
12096                    out.push(CommandOutcome {
12097                        command: command.clone(),
12098                        code: None,
12099                        output_tail: reason,
12100                        duration_ms: started.elapsed().as_millis() as u64,
12101                        resource_blocked: true,
12102                    });
12103                    break;
12104                }
12105                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12106                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12107                if let Some(failure) = failure {
12108                    body.push('\n');
12109                    body.push_str(&failure);
12110                }
12111                (code, body)
12112            }
12113            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12114        };
12115        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12116        if !body.is_empty() {
12117            // The full text is kept whatever the summary dropped. A fresh
12118            // name per command so a resume or retry never overwrites an
12119            // earlier run's output. A failed write changes no verdict.
12120            match crate::run::write_verification_artifact(state, task, &body) {
12121                Ok(name) if summary != body => {
12122                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12123                }
12124                Ok(_) => {}
12125                Err(e) => {
12126                    let warning = format!("could not save the full {task} output: {e:#}");
12127                    tracing::warn!("{warning}");
12128                    state.event(node, warning);
12129                    if summary != body {
12130                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12131                    }
12132                }
12133            }
12134        }
12135        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12136        out.push(CommandOutcome {
12137            command: command.clone(),
12138            code,
12139            output_tail: summary,
12140            duration_ms: started.elapsed().as_millis() as u64,
12141            resource_blocked,
12142        });
12143        if resource_blocked {
12144            state.resource_block = Some(crate::run::ResourceBlock {
12145                cause: crate::run::ResourceBlockCause::Other,
12146                node: node.to_owned(),
12147                kind: "incremental cache admission".into(),
12148                detail: body,
12149                at: Timestamp::now(),
12150            });
12151            break;
12152        }
12153    }
12154    state.task_finished(task);
12155    if let Err(e) = state.save() {
12156        tracing::warn!("could not persist the end of {task}: {e:#}");
12157    }
12158    (out, timed_out_pids)
12159}
12160
12161/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12162/// section (`report::run`) and in the `merge` event this node records — for
12163/// the operator to run by hand.
12164///
12165/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12166/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12167/// merge commits", or "require linear history") rejects the push a `--no-ff`
12168/// merge would produce, which is exactly the guidance this function replaces.
12169/// `message`'s first line becomes the squash commit's subject, matching the
12170/// note `report::run` prints alongside this command — see that function for
12171/// why an explicit subject is not optional there.
12172fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12173    let repo = repo.display();
12174    match style {
12175        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12176        MergeStyle::Squash => {
12177            // The subject sits inside double quotes, and a title an agent
12178            // wrote may carry the characters that break out of them.
12179            let subject = message
12180                .lines()
12181                .next()
12182                .unwrap_or(branch)
12183                .replace(['\\', '"', '$', '`'], "");
12184            format!(
12185                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12186            )
12187        }
12188        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12189    }
12190}
12191
12192/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12193/// under the hood, rejects a `title` over 256 characters and the whole
12194/// command fails — no PR at all, for a run whose body was otherwise fine
12195/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12196/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12197/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12198/// this bound. It is a margin, not a guarantee — a title packed
12199/// with multi-unit characters could still in principle land close to the
12200/// edge, but a real task title's occasional emoji or accented letter fits
12201/// comfortably inside it.
12202const PR_TITLE_MAX: usize = 240;
12203
12204/// A pull request title from the opening line of `text`, or `None` when that
12205/// line is not English (GitHub text is) or has no letters.
12206///
12207/// A line within `max` is kept as is. A longer one is cut at the end of its
12208/// first sentence when that falls inside `max`, else at a word boundary with a
12209/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12210/// make every merely-truncated title look non-English). The language check
12211/// runs on the kept text before any mark is added, so only what GitHub will
12212/// show is judged: an English opening followed by non-ASCII far past the cut
12213/// still passes.
12214fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12215    let line = queue::first_line(text)?;
12216    let chars: Vec<char> = line.chars().collect();
12217    let (kept, mark) = if chars.len() <= max {
12218        (line.to_owned(), "")
12219    } else if let Some(end) = sentence_end(&chars, max) {
12220        (chars[..end].iter().collect::<String>(), "")
12221    } else {
12222        let room = max.saturating_sub(3);
12223        // Cut at the last space inside the room; when the char just past the
12224        // room is a space the room already ends on a word.
12225        let cut = if chars[room].is_whitespace() {
12226            room
12227        } else {
12228            chars[..room]
12229                .iter()
12230                .rposition(|c| c.is_whitespace())
12231                .unwrap_or(room)
12232        };
12233        let head: String = chars[..cut].iter().collect();
12234        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12235        (head.to_owned(), "...")
12236    };
12237    if kept.is_empty() {
12238        return None;
12239    }
12240    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12241        return None;
12242    }
12243    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12244        return None;
12245    }
12246    Some(format!("{kept}{mark}"))
12247}
12248
12249/// The char length of the first sentence of `chars` when it ends within `max`
12250/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12251/// abbreviations so `e.g. foo` does not end a title early.
12252fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12253    const MIN: usize = 20;
12254    for i in MIN..max.min(chars.len()) {
12255        if !matches!(chars[i], '.' | '!' | '?') {
12256            continue;
12257        }
12258        let Some(&next) = chars.get(i + 1) else {
12259            continue;
12260        };
12261        if !next.is_whitespace() {
12262            continue;
12263        }
12264        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12265        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12266            continue;
12267        }
12268        let word: String = chars[..i]
12269            .iter()
12270            .rev()
12271            .take_while(|c| !c.is_whitespace())
12272            .collect::<Vec<_>>()
12273            .into_iter()
12274            .rev()
12275            .collect();
12276        let word = word.to_ascii_lowercase();
12277        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12278            continue;
12279        }
12280        let end = chars[..i]
12281            .iter()
12282            .rposition(|c| !c.is_whitespace())
12283            .map_or(i, |p| p + 1);
12284        return Some(end);
12285    }
12286    None
12287}
12288
12289/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12290/// change: a title and a body describing what was *implemented*, not the task
12291/// that asked for it. A task reads as a request; a reader of the merged
12292/// history wants the change.
12293struct PrMessage {
12294    title: String,
12295    body: String,
12296}
12297
12298impl PrMessage {
12299    /// Title, blank line, body. The first line is the squash/merge commit
12300    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12301    /// has to stay one sensible line.
12302    fn commit_message(&self) -> String {
12303        format!("{}\n\n{}", self.title, self.body)
12304    }
12305}
12306
12307/// The text after a leading `TITLE:` (any case) on `line`.
12308fn title_marker(line: &str) -> Option<&str> {
12309    let line = line.trim();
12310    let head = line.get(..6)?;
12311    head.eq_ignore_ascii_case("title:")
12312        .then(|| line[6..].trim())
12313}
12314
12315/// The implementer's own one-line title: the `TITLE:` line the implement
12316/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12317/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12318/// source, and a title that says as much is refused here too.
12319fn summary_title(summary: &str) -> Option<String> {
12320    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12321    let raw = title_marker(first)?;
12322    if raw.is_empty() {
12323        return None;
12324    }
12325    let title = queue::title_from(raw, PR_TITLE_MAX);
12326    let lower = title.to_ascii_lowercase();
12327    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12328        return None;
12329    }
12330    Some(title)
12331}
12332
12333/// How `open_review`'s instruction begins; see [`landing_title`].
12334const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12335
12336/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12337pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12338
12339/// Does this run review work that already existed, rather than implement a
12340/// task? Runs recorded before `reviewed_commits` existed carry only the
12341/// candidate marker.
12342pub(crate) fn is_review_run(state: &RunState) -> bool {
12343    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12344}
12345
12346/// The title of a review-only run: the subject of the oldest commit under
12347/// review. Later commits are usually fixups, and `instruction` is the review
12348/// prompt, which says nothing about the change. GitHub text is English, so a
12349/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12350fn review_title(state: &RunState) -> Option<String> {
12351    english_subject(
12352        state.reviewed_commits.as_ref()?.first()?,
12353        state.config.graph.github_english(),
12354    )
12355}
12356
12357/// `raw` as a pull request title, or `None` when it is blank, not English
12358/// (GitHub text is), or one of magi's own candidate commit subjects.
12359fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12360    let raw = raw.trim();
12361    if raw.is_empty() {
12362        return None;
12363    }
12364    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12365        return None;
12366    }
12367    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12368        return None;
12369    }
12370    let title = queue::title_from(raw, PR_TITLE_MAX);
12371    let lower = title.to_ascii_lowercase();
12372    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12373        return None;
12374    }
12375    Some(title)
12376}
12377
12378/// What a review-only run's branch says about itself, read at the moment the
12379/// pull request is opened.
12380#[derive(Debug, Clone, PartialEq, Eq)]
12381struct BranchFacts {
12382    /// `(subject, body)` of each commit, oldest first.
12383    commits: Vec<(String, String)>,
12384    /// Trimmed `git diff --stat`.
12385    stat: String,
12386}
12387
12388/// Longest diff stat shown: this many file lines plus the summary line.
12389const STAT_FILE_LINES: usize = 25;
12390/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12391const COMMITS_MAX_CHARS: usize = 20_000;
12392
12393/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12394/// say or finds nothing, so the caller falls back to what the run recorded.
12395async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12396    let commits = git::commit_log(repo, base, branch).await.ok()?;
12397    if commits.is_empty() {
12398        return None;
12399    }
12400    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12401    let lines: Vec<&str> = stat.lines().collect();
12402    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12403        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12404        let more = format!(" ... {omitted} more file(s)");
12405        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12406        kept.push(&more);
12407        kept.push(lines[lines.len() - 1]);
12408        kept.join("\n")
12409    } else {
12410        lines.join("\n")
12411    };
12412    Some(BranchFacts { commits, stat })
12413}
12414
12415/// Defang what would break the surrounding markdown: a closing `</details>`
12416/// and a code fence.
12417fn markdown_safe(text: &str) -> String {
12418    text.replace("</details>", "&lt;/details&gt;")
12419        .replace("\x60\x60\x60", "~~~")
12420}
12421
12422fn neutral_title(state: &RunState, winner: char) -> String {
12423    format!(
12424        "chore: land candidate {} of run {}",
12425        winner.to_ascii_uppercase(),
12426        state.id
12427    )
12428}
12429
12430/// Reuse the graph's bounded fixer without starting another graph or driver.
12431/// Copy the working state back even on errors so spent rounds remain visible.
12432pub(crate) async fn repair_landing_gate(
12433    state: &mut RunState,
12434    winner: &Candidate,
12435    outcomes: &[CommandOutcome],
12436) -> Result<GateFix> {
12437    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12438        return Ok(GateFix::Stop);
12439    }
12440    let mut runner = Runner {
12441        roles: state.config.resolve_roles()?,
12442        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12443        state: state.clone(),
12444        pause: Pause::new(),
12445        interrupt: Pause::new(),
12446    };
12447    let result = runner.gate_fix_round(winner, outcomes).await;
12448    *state = runner.state;
12449    result
12450}
12451
12452/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12453/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12454pub(crate) async fn verify_current_head(
12455    state: &mut RunState,
12456    winner: &Candidate,
12457    wait: &crate::repo_lock::WaitPolicy,
12458) -> Result<bool> {
12459    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12460    // Preserve the previous record before replacing it, including on crash recovery.
12461    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12462        state
12463            .verification_history
12464            .push(crate::run::VerificationEvidence {
12465                superseded_by: Some(head.clone()),
12466                archived_at: Some(Timestamp::now()),
12467                fixes_spent: Some(state.gate_fixes.len()),
12468                gate_head: None,
12469                gate_ran: false,
12470                gate: Vec::new(),
12471                e2e: state.head_e2e.take(),
12472            });
12473    }
12474    if let Some(e) = &state.head_e2e
12475        && e.head == head
12476    {
12477        if e.passed {
12478            return Ok(true);
12479        }
12480        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12481            state.status = RunStatus::Blocked;
12482            state.save()?;
12483            return Ok(false);
12484        }
12485    } else if state.reviews.last().is_some_and(|r| {
12486        r.verified_head.as_deref() == Some(head.as_str())
12487            && (r.e2e_status() == E2eStatus::Passed
12488                || (r.e2e_status() == E2eStatus::NotConfigured
12489                    && state.config.verify.e2e.is_empty()))
12490    }) || state
12491        .gate_repair_e2e
12492        .as_ref()
12493        .is_some_and(|e| e.head == head && e.passed)
12494    {
12495        return Ok(true);
12496    }
12497    if let Some(e) = &state.gate_repair_e2e
12498        && e.head == head
12499        && !e.passed
12500        && !e.outcomes.is_empty()
12501        && !verify_inconclusive(&e.outcomes)
12502    {
12503        state.status = RunStatus::Blocked;
12504        state.save()?;
12505        return Ok(false);
12506    }
12507    state.head_e2e = Some(crate::run::GateRepairE2e {
12508        head: head.clone(),
12509        passed: false,
12510        outcomes: Vec::new(),
12511    });
12512    state.save()?;
12513    let commands = state.config.verify.e2e.clone();
12514    let shell = state.config.shell();
12515    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12516    let cache_dir = state.config.cache_dir();
12517    let outcomes = if commands.is_empty() {
12518        Vec::new()
12519    } else {
12520        let (outcomes, _) = with_cache_lease(
12521            state,
12522            cache_dir.as_deref(),
12523            "e2e",
12524            "e2e",
12525            &winner.worktree,
12526            &head,
12527            timeout,
12528            "current head e2e",
12529            wait,
12530            |state, budget| async move {
12531                run_e2e_with_retry(
12532                    state,
12533                    &shell,
12534                    &commands,
12535                    &winner.worktree,
12536                    budget,
12537                    "current head e2e",
12538                )
12539                .await
12540            },
12541        )
12542        .await?;
12543        outcomes
12544    };
12545    let unresolved = verify_inconclusive(&outcomes);
12546    let passed = !unresolved
12547        && outcomes.iter().all(CommandOutcome::ok)
12548        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12549    if !outcomes.is_empty() && !unresolved {
12550        state.record_base_progress(&head);
12551    }
12552    state.head_e2e = Some(crate::run::GateRepairE2e {
12553        head: head.clone(),
12554        passed,
12555        outcomes,
12556    });
12557    if !passed && !unresolved {
12558        state.status = RunStatus::Blocked;
12559        state.event(
12560            "e2e",
12561            format!("verify.e2e failed on head {head}; nothing published"),
12562        );
12563    }
12564    state.save()?;
12565    Ok(passed)
12566}
12567
12568/// Recheck a repair's e2e requirement across failures, resource deferrals and
12569/// process interruptions. A successful result only applies to its exact SHA.
12570pub(crate) async fn check_gate_repair_e2e(
12571    state: &mut RunState,
12572    winner: &Candidate,
12573    wait: &crate::repo_lock::WaitPolicy,
12574) -> Result<GateFix> {
12575    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12576        return Ok(GateFix::Retry);
12577    }
12578    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12579    if state
12580        .gate_repair_e2e
12581        .as_ref()
12582        .is_some_and(|r| r.head == head && r.passed)
12583    {
12584        return Ok(GateFix::Retry);
12585    }
12586    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12587        head: head.clone(),
12588        passed: false,
12589        outcomes: Vec::new(),
12590    });
12591    state.save()?;
12592    let commands = state.config.verify.e2e.clone();
12593    let outcomes = if commands.is_empty() {
12594        Vec::new()
12595    } else {
12596        let shell = state.config.shell();
12597        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12598        let cache_dir = state.config.cache_dir();
12599        let (outcomes, _) = with_cache_lease(
12600            state,
12601            cache_dir.as_deref(),
12602            "e2e",
12603            "e2e",
12604            &winner.worktree,
12605            &head,
12606            timeout,
12607            "gate repair e2e",
12608            wait,
12609            |state, budget| async move {
12610                run_e2e_with_retry(
12611                    state,
12612                    &shell,
12613                    &commands,
12614                    &winner.worktree,
12615                    budget,
12616                    "gate repair e2e",
12617                )
12618                .await
12619            },
12620        )
12621        .await?;
12622        outcomes
12623    };
12624    let deferred = verify_inconclusive(&outcomes);
12625    let passed = outcomes.iter().all(CommandOutcome::ok)
12626        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12627    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12628        head,
12629        passed,
12630        outcomes,
12631    });
12632    state.save()?;
12633    if deferred {
12634        Ok(GateFix::Defer)
12635    } else if !passed {
12636        state.event(
12637            "gate",
12638            "verify.e2e failed after gate repair; nothing published",
12639        );
12640        Ok(GateFix::Stop)
12641    } else {
12642        Ok(GateFix::Retry)
12643    }
12644}
12645
12646/// Run the same configured gate for a landing head and a fresh graph.
12647pub(crate) async fn run_landing_gate(
12648    state: &mut RunState,
12649    winner: &Candidate,
12650    wait: &crate::repo_lock::WaitPolicy,
12651) -> Result<Vec<CommandOutcome>> {
12652    let shell = state.config.shell();
12653    let gate_commands = state.config.verify.gate.clone();
12654    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12655    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12656        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12657        Vec::new()
12658    } else {
12659        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12660        let cache_dir = state.config.cache_dir();
12661        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12662        let (outcomes, _) = with_cache_lease(
12663            state,
12664            cache_dir.as_deref(),
12665            "gate",
12666            "gate",
12667            &winner.worktree,
12668            &head,
12669            timeout,
12670            "final gate",
12671            wait,
12672            |state, budget| {
12673                let shell = shell.clone();
12674                let gate_commands = gate_commands.clone();
12675                let winner = winner.clone();
12676                async move {
12677                    Runner::run_pre_gate(state, &winner).await;
12678                    // Bind command evidence after mechanical fixes, before commands start.
12679                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12680                    let (outcomes, timed_out_pids) = run_commands(
12681                        state,
12682                        "gate",
12683                        "gate",
12684                        0,
12685                        &shell,
12686                        &gate_commands,
12687                        &winner.worktree,
12688                        budget,
12689                    )
12690                    .await;
12691                    (outcomes, false, timed_out_pids)
12692                }
12693            },
12694        )
12695        .await?;
12696        outcomes
12697    };
12698    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12699    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12700        let mut outcomes = outcomes;
12701        outcomes.push(CommandOutcome {
12702            command: "gate head stability".into(), code: None,
12703            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12704            duration_ms: 0, resource_blocked: false,
12705        });
12706        return Ok(outcomes);
12707    }
12708    if outcomes.is_empty() {
12709        // Nothing configured to check — distinct from every other
12710        // silence in this run's event log, since an empty `gate` alone
12711        // no longer says whether the gate ran at all (see
12712        // `RunState::gate_ran`'s own doc).
12713        state.event("gate", "no gate commands configured; not tested");
12714    }
12715    for o in &outcomes {
12716        state.event(
12717            "gate",
12718            format!(
12719                "`{}` -> {}",
12720                o.command,
12721                if o.ok() {
12722                    "pass".to_owned()
12723                } else {
12724                    format!(
12725                        "FAIL ({:?})\n{}",
12726                        o.code,
12727                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12728                    )
12729                }
12730            ),
12731        );
12732    }
12733    Ok(outcomes)
12734}
12735
12736/// The pull request title to hand to `land::merge_subject`. A review-only run
12737/// opened by an earlier build titled its pull request with the review prompt;
12738/// that title is dropped (empty, so the fallback applies) rather than landed.
12739/// Any other title, including an operator's rename, passes through untouched,
12740/// and so does every title of a run that implements a task.
12741pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12742    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12743        ""
12744    } else {
12745        pr_title
12746    }
12747}
12748
12749/// What the squash subject falls back to when the pull request title is empty
12750/// or candidate-shaped: for a review-only run the derived title, never the
12751/// review prompt held in `instruction`.
12752pub fn landing_subject_source(state: &RunState) -> String {
12753    if is_review_run(state) {
12754        let winner = state.candidates.first().map_or('A', |c| c.label);
12755        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12756    }
12757    state.instruction.clone()
12758}
12759
12760/// `summary` without its `TITLE:` line, which the pull request title already
12761/// carries.
12762fn summary_without_title(summary: &str) -> String {
12763    let mut lines = summary.trim().lines().peekable();
12764    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12765        lines.next();
12766    }
12767    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12768}
12769
12770/// The pull request title and body for the winning candidate.
12771///
12772/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12773/// the task's own opening line via [`queue::title_from`] when there is none.
12774/// `state.instruction` can open with blank lines (`task_text` only rejects a
12775/// body that is blank *entirely*), which `title_from` skips.
12776///
12777/// Publishing uses an independent description generated from the final diff
12778/// and structured evidence. Internal summaries and reports remain local.
12779/// Non-publishing previews may display the original task and candidate summary.
12780/// The footer retains the run and candidate provenance.
12781#[cfg(test)]
12782fn pr_message(state: &RunState, winner: char) -> PrMessage {
12783    pr_message_with(state, winner, None)
12784}
12785
12786/// [`pr_message`] with what the branch of a review-only run says about itself.
12787/// `facts` is ignored for a run that implements a task.
12788#[cfg(test)]
12789fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12790    let raw = pr_message_raw(
12791        state,
12792        winner,
12793        facts,
12794        crate::pr_outcome::FinalHead::Unreported,
12795        ChangeNote::NotAsked,
12796    );
12797    let id = crate::scrub::Identity::current();
12798    PrMessage {
12799        title: crate::scrub::scrub(&raw.title, &id),
12800        body: crate::scrub::scrub(&raw.body, &id),
12801    }
12802}
12803
12804/// What the public description may say about the change itself.
12805#[derive(Debug, Clone, Copy)]
12806enum ChangeNote<'a> {
12807    /// A local preview or non-publishing merge mode: no public generation needed.
12808    NotAsked,
12809    /// A summary written from the final diff.
12810    Summary(&'a crate::run::ChangeSummary),
12811    /// A summary was asked for and could not be produced; the text says why.
12812    Unavailable(&'a str),
12813}
12814
12815fn pr_message_raw(
12816    state: &RunState,
12817    winner: char,
12818    facts: Option<&BranchFacts>,
12819    head: crate::pr_outcome::FinalHead<'_>,
12820    change: ChangeNote<'_>,
12821) -> PrMessage {
12822    if !matches!(change, ChangeNote::NotAsked) {
12823        let (title, mut body) = match change {
12824            ChangeNote::Summary(c) => (
12825                c.title.clone(),
12826                format!("## Summary\n\n{}\n\n", c.body.trim()),
12827            ),
12828            ChangeNote::Unavailable(why) => (
12829                neutral_title(state, winner),
12830                format!(
12831                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12832                ),
12833            ),
12834            ChangeNote::NotAsked => unreachable!(),
12835        };
12836        body.push_str(&public_pr_evidence(state, head));
12837        body.push_str(&format!(
12838            "\n---\nmagi:run/{} magi:candidate-{}\n",
12839            state.id,
12840            winner.to_ascii_lowercase()
12841        ));
12842        return PrMessage { title, body };
12843    }
12844    pr_preview(state, winner, facts, head)
12845}
12846
12847/// Append public facts from the run records after every prose generation,
12848/// including posting-gate rewrites. The agent cannot drop this provenance.
12849fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12850    let mut body = crate::pr_outcome::review_rounds(state);
12851    body.push_str(&crate::pr_outcome::final_verification(state, head));
12852    let open = state.last_round_findings();
12853    if !open.is_empty() {
12854        body.push_str("## Open review findings\n\n");
12855        for f in open {
12856            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12857        }
12858        body.push('\n');
12859    }
12860    if let Some(c) = state.change_summary.as_ref()
12861        && head == crate::pr_outcome::FinalHead::At(&c.head)
12862    {
12863        body.push_str(&format!(
12864            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12865            c.base, c.head
12866        ));
12867    }
12868    body
12869}
12870
12871/// Local-only preview of the operator's task and candidate summary.
12872fn pr_preview(
12873    state: &RunState,
12874    winner: char,
12875    facts: Option<&BranchFacts>,
12876    head: crate::pr_outcome::FinalHead<'_>,
12877) -> PrMessage {
12878    let summary = state
12879        .candidates
12880        .iter()
12881        .find(|c| c.label == winner)
12882        .map(|c| c.summary.as_str())
12883        .unwrap_or_default();
12884    // The fallback is the operator's own words and may not be English; GitHub
12885    // text always is, so a non-English task gets a neutral title instead.
12886    let review = is_review_run(state);
12887    let title = if review {
12888        facts
12889            .and_then(|f| {
12890                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12891            })
12892            .or_else(|| review_title(state))
12893            .or_else(|| {
12894                state
12895                    .candidates
12896                    .iter()
12897                    .find(|c| c.label == winner)
12898                    .filter(|c| !c.branch.starts_with("magi/"))
12899                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12900            })
12901            .unwrap_or_else(|| neutral_title(state, winner))
12902    } else {
12903        summary_title(summary).unwrap_or_else(|| {
12904            english_title(
12905                &state.instruction,
12906                PR_TITLE_MAX,
12907                state.config.graph.github_english(),
12908            )
12909            .unwrap_or_else(|| neutral_title(state, winner))
12910        })
12911    };
12912
12913    let mut body = String::new();
12914    let what = summary_without_title(summary);
12915    if !what.is_empty() {
12916        body.push_str("## Summary\n\n");
12917        body.push_str(&what);
12918        body.push_str("\n\n");
12919    }
12920
12921    // Review history uses structured facts across all rounds, never reports.
12922    body.push_str(&crate::pr_outcome::review_rounds(state));
12923    body.push_str(&crate::pr_outcome::final_verification(state, head));
12924    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12925
12926    let open = state.open_findings();
12927    if !open.is_empty() {
12928        body.push_str("## Open review findings\n\n");
12929        for f in &open {
12930            body.push_str(&format!(
12931                "- `{}` [{:?}] {}\n",
12932                f.id, f.severity, "Context retained in local review records."
12933            ));
12934        }
12935        body.push('\n');
12936    }
12937
12938    if let Some(fix) = fix
12939        && !fix.rejected.is_empty()
12940    {
12941        body.push_str("## Declined by the fixer\n\n");
12942        for r in &fix.rejected {
12943            body.push_str(&format!(
12944                "- `{}`: {}\n",
12945                r.id, "Reason retained in local review records."
12946            ));
12947        }
12948        body.push('\n');
12949    }
12950
12951    if review {
12952        // The review prompt is not the task; list what the branch carries.
12953        body.push_str("## Commits under review\n\n");
12954        if let Some(facts) = facts {
12955            let mut left = COMMITS_MAX_CHARS;
12956            for (i, (subject, text)) in facts.commits.iter().enumerate() {
12957                let mut entry = format!("- {}\n", markdown_safe(subject));
12958                for l in markdown_safe(text).lines() {
12959                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
12960                }
12961                if left == 0 {
12962                    body.push_str(&format!(
12963                        "- ... {} more commit(s)\n",
12964                        facts.commits.len() - i
12965                    ));
12966                    break;
12967                }
12968                if entry.len() > left {
12969                    // Even the first commit is cut: one huge body must not
12970                    // push the whole description past GitHub's limit.
12971                    let mut end = left;
12972                    while !entry.is_char_boundary(end) {
12973                        end -= 1;
12974                    }
12975                    entry.truncate(end);
12976                    entry.push_str("\n  ... (truncated)\n");
12977                    left = 0;
12978                } else {
12979                    left -= entry.len();
12980                }
12981                body.push_str(&entry);
12982            }
12983            if !facts.stat.trim().is_empty() {
12984                body.push_str(&format!(
12985                    "\n## Diff stat\n\n```\n{}\n```\n",
12986                    markdown_safe(facts.stat.trim())
12987                ));
12988            }
12989        } else {
12990            match &state.reviewed_commits {
12991                Some(subjects) => {
12992                    for s in subjects {
12993                        body.push_str(&format!("- {}\n", s.trim()));
12994                    }
12995                }
12996                None => {
12997                    // An older run kept only the prompt, with the commit list
12998                    // after its first paragraph.
12999                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13000                    body.push_str(rest.trim());
13001                    body.push('\n');
13002                }
13003            }
13004        }
13005    } else {
13006        let task = state.instruction.trim();
13007        let task = if task.is_empty() {
13008            "(empty task)"
13009        } else {
13010            task
13011        };
13012        body.push_str(&format!(
13013            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13014            task.replace("</details>", "&lt;/details&gt;")
13015        ));
13016    }
13017
13018    body.push_str(&format!(
13019        "\n---\nmagi:run/{} magi:candidate-{}\n",
13020        state.id,
13021        winner.to_ascii_lowercase()
13022    ));
13023
13024    PrMessage { title, body }
13025}
13026
13027/// Source imports have already been prepared from structured inputs. Legacy
13028/// inferred references must never instruct a seat to reconstruct those imports.
13029fn seeded_instruction(state: &RunState) -> String {
13030    let mut inputs = Vec::new();
13031    if let Some(start) = &state.sources.start_from {
13032        inputs.push(format!(
13033            "Start from `{}` at `{}` (including ancestry).",
13034            start.token, start.sha
13035        ));
13036    }
13037    for pick in &state.sources.pick_commits {
13038        inputs.push(format!(
13039            "Pick `{}` at `{}` (single commit).",
13040            pick.token, pick.sha
13041        ));
13042    }
13043    if inputs.is_empty() {
13044        state.instruction.clone()
13045    } else {
13046        format!(
13047            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13048            state.instruction,
13049            inputs.join("\n")
13050        )
13051    }
13052}
13053
13054/// Does the winner have no commits ahead of the base it would land on?
13055/// Any failure to find out reads as "not empty": the merge then behaves as it
13056/// always did rather than refusing on a guess.
13057async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13058    let base = &state.base_branch;
13059    let mut against = base.clone();
13060    if mode == MergeMode::Pr {
13061        // A pull request lands on the remote's base, never the local branch
13062        // of the same name: if that cannot be read, "not empty" is the safe
13063        // answer.
13064        let remote = &state.config.merge.remote;
13065        let tracking = format!("{remote}/{base}");
13066        let fetched = git::fetch(repo, remote, base).await;
13067        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13068            against = tracking;
13069        } else {
13070            return false;
13071        }
13072    }
13073    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13074}
13075
13076/// Why nothing was opened for an empty winner, with what the task's own
13077/// references resolved to.
13078fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13079    let mut detail = format!(
13080        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13081         nothing to open a pull request for"
13082    );
13083    detail.push_str(
13084        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13085    );
13086    detail
13087}
13088
13089/// What the `Pr` merge does once it knows whether the branch already has an
13090/// open pull request.
13091#[derive(Debug, PartialEq, Eq)]
13092enum PrPlan {
13093    Create,
13094    Adopt { url: String, title: String },
13095    Stop(String),
13096}
13097
13098/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13099/// a failed lookup -> stop with the real reason. Never guesses.
13100fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13101    match found {
13102        Ok(land::OpenPr::None) => PrPlan::Create,
13103        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13104        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13105            "several open pull requests exist for this branch, not picking one: {}",
13106            urls.join(" ")
13107        )),
13108        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13109    }
13110}
13111
13112/// `gh pr create`, returning the PR url.
13113pub(crate) async fn gh_pr_create(
13114    cwd: &Path,
13115    base: &str,
13116    head: &str,
13117    title: &str,
13118    body: &str,
13119) -> Result<String> {
13120    let out = tokio::process::Command::new("gh")
13121        .args([
13122            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13123        ])
13124        .current_dir(cwd)
13125        .quiet()
13126        .stdin(std::process::Stdio::null())
13127        .output()
13128        .await
13129        .context("spawn gh")?;
13130    if out.status.success() {
13131        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13132    } else {
13133        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13134    }
13135}
13136
13137/// Never force deletion of uncommitted work, including unknown legacy trees.
13138async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13139    match std::fs::symlink_metadata(path) {
13140        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13141            // A previous cleanup may have removed the tree before a branch
13142            // deletion or state save failed. Unregister it if still present.
13143            git::worktree_remove_clean(repo, path).await?;
13144            return Ok(());
13145        }
13146        Err(e) => return Err(e.into()),
13147        Ok(_) => {}
13148    }
13149    if !git::is_clean(path).await? {
13150        bail!(
13151            "worktree has uncommitted work; refusing removal: {}",
13152            path.display()
13153        );
13154    }
13155    if !git::worktree_remove_clean(repo, path).await? {
13156        bail!("worktree removal refused: {}", path.display());
13157    }
13158    Ok(())
13159}
13160
13161/// Tear a run's worktrees and branches down.
13162///
13163/// `home` is where the updated `run.json` is saved (via
13164/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13165/// a housekeeping pass already has its own honest `home` handed to it, and
13166/// falling through to the global here would write back through whichever
13167/// directory some other process or test pinned into that `OnceLock` first,
13168/// not the one the caller actually resolved its `runs` and `state` from.
13169pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13170    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13171        bail!("refusing fold: {reason}");
13172    }
13173    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13174        bail!("candidate preservation incomplete; refusing cleanup");
13175    }
13176    // Serialize ordinary folding with the same starts/resumes as target-only
13177    // eviction. Otherwise a second janitor could remove the whole worktree
13178    // around a live output borrower even though target eviction was fenced.
13179    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13180    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13181    let _driver = crate::driver::maintenance(home, &state.id)?;
13182    let mut output_guards = Vec::new();
13183    for entry in std::fs::read_dir(state.worktree_root())
13184        .into_iter()
13185        .flatten()
13186        .flatten()
13187    {
13188        if entry.path().join(".git").is_file() {
13189            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13190        }
13191    }
13192    let repo = state.repo.clone();
13193    // CLI, Web and automatic cleanup all enter here, independently of
13194    // Runner::fold_losers. Protect deletion and shared config release too.
13195    let wait = repository_wait(state, None);
13196    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13197        Ok(lease) => lease,
13198        Err(e) => {
13199            state.save_under(home)?;
13200            return Err(e);
13201        }
13202    };
13203    // Confirmed merges dispose of their owned sources after durable evidence
13204    // capture. Preflight every source before removing anything.
13205    let _source_outputs = if state.status == RunStatus::Merged
13206        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13207    {
13208        verify_merged_fold_sources(state, drop_winner, home).await?
13209    } else {
13210        Vec::new()
13211    };
13212    let root = state.worktree_root();
13213    let winner = state.tally.as_ref().map(|t| t.winner);
13214    let mut removed = Vec::new();
13215
13216    for i in 0..state.candidates.len() {
13217        let c = state.candidates[i].clone();
13218        let is_winner = Some(c.label) == winner;
13219        if is_winner && !drop_winner {
13220            continue;
13221        }
13222        let existed = c.worktree.exists();
13223        // A vanished directory can still have a Git registration that keeps
13224        // its branch checked out. Remove that registration before the branch.
13225        remove_preserved_worktree(&repo, &c.worktree).await?;
13226        if existed {
13227            removed.push(c.worktree.to_string_lossy().into_owned());
13228        }
13229        // A branch handed to a later run (and its pull request) is not this
13230        // run's to delete.
13231        let handed_over = state.released_branches.contains(&c.branch);
13232        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13233            if !git::branch_delete(&repo, &c.branch).await? {
13234                bail!("candidate branch removal refused");
13235            }
13236            removed.push(c.branch.clone());
13237        }
13238        state.candidates[i].folded = true;
13239    }
13240
13241    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13242        let path = name.path();
13243        let keep = !drop_winner
13244            && winner.is_some_and(|w| {
13245                path.file_name()
13246                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13247            });
13248        if keep {
13249            continue;
13250        }
13251        remove_preserved_worktree(&repo, &path).await?;
13252        removed.push(path.to_string_lossy().into_owned());
13253    }
13254
13255    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13256    // judge worktrees, so once the loop above has cleared all of them out,
13257    // the parent is a bare directory nobody else was ever going to remove -
13258    // git only ever managed what was inside it. Left alone, one of these
13259    // accumulates per fully-folded run; the operator's own machine had 74.
13260    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13261    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13262    // so does anything a run never claimed that happens to share the bay.
13263    remove_if_empty(&root);
13264
13265    if state.enabled_worktree_config && drop_winner {
13266        // A release, not a raw disable: some sibling run in this repository
13267        // may still hold its own reference (see `git::acquire_worktree_config`),
13268        // and only the last release actually turns the setting back off.
13269        git::release_worktree_config(&repo).await.ok();
13270        state.enabled_worktree_config = false;
13271    }
13272    state.save_under(home)?;
13273    Ok(removed)
13274}
13275
13276async fn verify_merged_fold_sources(
13277    state: &RunState,
13278    drop_winner: bool,
13279    home: &Path,
13280) -> Result<Vec<std::fs::File>> {
13281    let winner = state.tally.as_ref().map(|t| t.winner);
13282    let mut heads = BTreeSet::new();
13283    for candidate in &state.candidates {
13284        if Some(candidate.label) == winner && !drop_winner {
13285            continue;
13286        }
13287        anyhow::ensure!(
13288            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13289            "candidate recovery source is outside the inventoried run root: {}",
13290            candidate.worktree.display()
13291        );
13292        match std::fs::symlink_metadata(&candidate.worktree) {
13293            Ok(_) => {}
13294            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13295            Err(error) => return Err(error.into()),
13296        }
13297        if !state.released_branches.contains(&candidate.branch)
13298            && git::branch_exists(&state.repo, &candidate.branch).await?
13299        {
13300            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13301        }
13302    }
13303    let mut cargo_guards = Vec::new();
13304    let entries = match std::fs::read_dir(state.worktree_root()) {
13305        Ok(entries) => Some(entries),
13306        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13307        Err(error) => return Err(error.into()),
13308    };
13309    for entry in entries.into_iter().flatten() {
13310        let path = entry?.path();
13311        let metadata = std::fs::symlink_metadata(&path)?;
13312        anyhow::ensure!(
13313            metadata.is_dir() && !metadata.file_type().is_symlink(),
13314            "non-directory or symlink recovery source retained: {}",
13315            path.display()
13316        );
13317        if !drop_winner
13318            && winner.is_some_and(|w| {
13319                path.file_name()
13320                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13321            })
13322        {
13323            continue;
13324        }
13325        anyhow::ensure!(
13326            path.join(".git").is_file(),
13327            "unregistered recovery source retained: {}",
13328            path.display()
13329        );
13330        anyhow::ensure!(
13331            git::git(
13332                &path,
13333                &[
13334                    "status",
13335                    "--porcelain",
13336                    "--untracked-files=all",
13337                    "--ignored",
13338                    "--",
13339                    ".",
13340                    ":(exclude)target"
13341                ]
13342            )
13343            .await?
13344            .is_empty(),
13345            "source changes or ignored recovery files retained: {}",
13346            path.display()
13347        );
13348        match std::fs::symlink_metadata(path.join("target")) {
13349            Ok(_) => {
13350                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13351            }
13352            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13353            Err(error) => return Err(error.into()),
13354        }
13355        for checkpoint in [
13356            "rebase-merge",
13357            "rebase-apply",
13358            "sequencer",
13359            "CHERRY_PICK_HEAD",
13360            "MERGE_HEAD",
13361            "REBASE_HEAD",
13362            "REVERT_HEAD",
13363            "BISECT_LOG",
13364            "AUTO_MERGE",
13365            "MERGE_MSG",
13366        ] {
13367            let location = git::git(
13368                &path,
13369                &[
13370                    "rev-parse",
13371                    "--path-format=absolute",
13372                    "--git-path",
13373                    checkpoint,
13374                ],
13375            )
13376            .await?;
13377            anyhow::ensure!(
13378                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13379                if e.kind() == std::io::ErrorKind::NotFound),
13380                "Git recovery checkpoint retained: {checkpoint}"
13381            );
13382        }
13383        heads.insert(git::rev_parse(&path, "HEAD").await?);
13384    }
13385    crate::fold_evidence::preserve(state, home, &heads).await?;
13386
13387    Ok(cargo_guards)
13388}
13389
13390/// Remove `dir` if it exists and has nothing in it.
13391///
13392/// Best-effort and silent by design: a directory that is not empty (a run
13393/// whose winner is still parked there, a stray file some other process left)
13394/// is exactly the case this must refuse, and a directory that is already gone
13395/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13396/// already refuses a non-empty directory, so the emptiness check below is
13397/// belt, not suspenders - it is what keeps this from ever attempting the
13398/// removal in the case that matters, rather than trusting `remove_dir`'s
13399/// error path to have no side effects if it ever changed.
13400fn remove_if_empty(dir: &Path) {
13401    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13402        std::fs::remove_dir(dir).ok();
13403    }
13404}
13405
13406/// Severity of the worst open finding in the last review round, for reporting.
13407pub fn worst_open(state: &RunState) -> Option<Severity> {
13408    state
13409        .reviews
13410        .last()?
13411        .reviews
13412        .iter()
13413        .flat_map(|r| r.findings.iter())
13414        .map(|f| f.severity)
13415        .max()
13416}
13417
13418/// `[worktree] setup` hides its products per worktree, which needs
13419/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13420/// worktrees, like the hook's; every place that runs setup calls this first.
13421async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13422    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13423        git::acquire_worktree_config(repo).await?;
13424        state.enabled_worktree_config = true;
13425    }
13426    Ok(())
13427}
13428
13429#[cfg(test)]
13430mod tests {
13431    #[tokio::test]
13432    async fn launcher_base_fetch_requires_remote_success() {
13433        let tmp = tempfile::tempdir().unwrap();
13434        let repo = tmp.path().join("repo");
13435        let remote = tmp.path().join("remote");
13436        std::fs::create_dir(&repo).unwrap();
13437        std::fs::create_dir(&remote).unwrap();
13438        git::git(&remote, &["init", "--bare", "-b", "main"])
13439            .await
13440            .unwrap();
13441        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13442        git::git(
13443            &repo,
13444            &[
13445                "-c",
13446                "user.name=Fixture",
13447                "-c",
13448                "user.email=fixture@example.invalid",
13449                "commit",
13450                "--allow-empty",
13451                "-m",
13452                "fixture",
13453            ],
13454        )
13455        .await
13456        .unwrap();
13457        git::git(
13458            &repo,
13459            &["remote", "add", "origin", remote.to_str().unwrap()],
13460        )
13461        .await
13462        .unwrap();
13463        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13464        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13465        assert_eq!(
13466            resolve_base(&repo, "main", "origin", None)
13467                .await
13468                .unwrap()
13469                .commit,
13470            head
13471        );
13472        #[cfg(unix)]
13473        {
13474            let policy: crate::config::Coordination = toml::from_str(
13475                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13476            ).unwrap();
13477            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13478            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13479            let held = crate::repo_lock::SeatLease::operation_with(
13480                &repo,
13481                Some(&policy),
13482                &crate::repo_lock::WaitPolicy::default(),
13483            )
13484            .await
13485            .unwrap();
13486            assert!(held.is_some());
13487            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13488            tokio::pin!(fetch);
13489            assert!(
13490                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13491                    .await
13492                    .is_err()
13493            );
13494            assert!(
13495                !repo.join(".git/FETCH_HEAD").exists(),
13496                "no fetch before admission"
13497            );
13498            drop(held);
13499            let fetched = fetch.await.unwrap();
13500            assert_eq!(fetched.commit, head);
13501            assert_eq!(fetched.remote, "origin");
13502            assert_eq!(fetched.branch, "main");
13503        }
13504        // A surviving tracking ref is not successful fetch evidence.
13505        git::git(
13506            &repo,
13507            &[
13508                "remote",
13509                "set-url",
13510                "origin",
13511                tmp.path().join("missing").to_str().unwrap(),
13512            ],
13513        )
13514        .await
13515        .unwrap();
13516        let error = resolve_base(&repo, "main", "origin", None)
13517            .await
13518            .unwrap_err()
13519            .to_string();
13520        assert!(error.contains("cannot read origin/main"));
13521        assert!(error.contains("refusing to branch off"));
13522    }
13523
13524    #[test]
13525    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13526        let leaked = vec!["chore(deps): update a crate".to_owned()];
13527        let own = "fix(daemon): apply a chosen action";
13528        assert!(should_retitle("", own, &leaked));
13529        assert!(should_retitle(
13530            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13531            own,
13532            &leaked
13533        ));
13534        assert!(should_retitle(
13535            "chore: land candidate A of run 1",
13536            own,
13537            &leaked
13538        ));
13539        assert!(should_retitle(
13540            "magi: candidate A (uncommitted work)",
13541            own,
13542            &leaked
13543        ));
13544        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13545        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13546        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13547    }
13548
13549    #[test]
13550    fn pr_merge_plan_creates_adopts_or_stops() {
13551        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13552        assert_eq!(
13553            pr_merge_plan(Ok(land::OpenPr::One {
13554                url: "u".into(),
13555                title: "t".into()
13556            })),
13557            PrPlan::Adopt {
13558                url: "u".into(),
13559                title: "t".into()
13560            }
13561        );
13562        let PrPlan::Stop(many) =
13563            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13564        else {
13565            panic!("many must stop");
13566        };
13567        assert!(many.contains('a') && many.contains('b'));
13568        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13569            panic!("a failed lookup must stop");
13570        };
13571        assert!(err.contains("bad token"));
13572    }
13573
13574    use super::*;
13575    use crate::run::GateStatus;
13576    use std::collections::BTreeMap;
13577    use std::time::Duration;
13578
13579    fn conductor() -> AgentSpec {
13580        AgentSpec {
13581            id: "conductor".to_owned(),
13582            kind: crate::config::AgentKind::Command,
13583            model: None,
13584            command: vec!["true".to_owned()],
13585            extra_args: Vec::new(),
13586            env: BTreeMap::new(),
13587            prompt_delivery: None,
13588            max_turn_secs: None,
13589        }
13590    }
13591
13592    fn spec(id: &str) -> AgentSpec {
13593        AgentSpec {
13594            id: id.to_owned(),
13595            kind: crate::config::AgentKind::Command,
13596            model: None,
13597            command: vec!["true".to_owned()],
13598            extra_args: Vec::new(),
13599            env: BTreeMap::new(),
13600            prompt_delivery: None,
13601            max_turn_secs: None,
13602        }
13603    }
13604
13605    fn ids(xs: &[&str]) -> BTreeSet<String> {
13606        xs.iter().map(|s| (*s).to_owned()).collect()
13607    }
13608
13609    #[test]
13610    fn pick_successor_skips_an_agent_another_seat_holds() {
13611        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13612        let roster = [spec("a"), spec("b"), spec("c")];
13613        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13614        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13615    }
13616
13617    #[test]
13618    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13619        // The other seat started on c but was handed to d; c is free again.
13620        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13621        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13622        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13623        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13624        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13625    }
13626
13627    #[test]
13628    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13629        let roster = [spec("a"), spec("b")];
13630        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13631        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13632        let carried = ids(&["b"]);
13633        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13634        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13635    }
13636
13637    #[test]
13638    fn pick_successor_returns_none_without_any_untried_successor() {
13639        let roster = [spec("a"), spec("b")];
13640        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13641        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13642        let carried = ids(&["a"]);
13643        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13644    }
13645
13646    #[test]
13647    fn pick_successor_rescue_avoids_another_seats_occupant() {
13648        // b is a carried failure and free; a is held by the other seat.
13649        let roster = [spec("a"), spec("b"), spec("c")];
13650        let carried = ids(&["b", "c"]);
13651        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13652        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13653    }
13654
13655    #[test]
13656    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13657        let roster = [spec("a"), spec("b"), spec("c")];
13658        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13659        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13660    }
13661
13662    #[test]
13663    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13664        let roster = [spec("a"), spec("b"), spec("c")];
13665        let failed = ids(&["a", "b", "c"]);
13666        // Rescue looks at the whole roster, once per id, then runs out.
13667        let mut tried = ids(&["b"]);
13668        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13669        assert_eq!(first.id, "a");
13670        tried.insert(first.id.clone());
13671        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13672        assert_eq!(second.id, "c");
13673        tried.insert(second.id.clone());
13674        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13675    }
13676
13677    #[test]
13678    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13679        let roster = [spec("a"), spec("b")];
13680        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13681        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13682    }
13683
13684    #[test]
13685    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13686        let roster = [spec("a"), spec("b"), spec("c")];
13687        let h = SeatHistory {
13688            failed: ids(&["a"]),
13689            last_ok: Some("b".to_owned()),
13690            last_fail: None,
13691        };
13692        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13693        // A last answerer that left the roster, or later failed, is ignored.
13694        let gone = SeatHistory {
13695            last_ok: Some("zzz".to_owned()),
13696            ..h.clone()
13697        };
13698        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13699        let failed = SeatHistory {
13700            failed: ids(&["a", "b"]),
13701            last_ok: Some("b".to_owned()),
13702            last_fail: None,
13703        };
13704        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13705    }
13706
13707    #[test]
13708    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13709        let first = SeatState::new("review-1", "alpha", 7);
13710        let next = handover_seat("review-1", "gamma", 7);
13711        assert_ne!(first.claude_session, next.claude_session);
13712    }
13713
13714    #[test]
13715    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13716        let mut state = state_with_summary("x", "y");
13717        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13718        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13719        assert_ne!(a.claude_session, b.claude_session);
13720    }
13721
13722    #[test]
13723    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13724        let roster = [spec("a"), spec("b")];
13725        let h = SeatHistory {
13726            failed: ids(&["a", "b"]),
13727            ..SeatHistory::default()
13728        };
13729        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13730        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13731        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13732    }
13733
13734    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13735    // fallback loop depends on to terminate: it must walk forward from the
13736    // seat's own position, never restart at the front of the roster, and it
13737    // must never hand back an id already tried, however many times that id
13738    // happens to appear.
13739
13740    #[test]
13741    fn failure_signature_ignores_numbers_and_paths() {
13742        assert_eq!(
13743            failure_signature("exited with Some(2) and no usable output"),
13744            failure_signature("exited with Some(137) and no usable output")
13745        );
13746        assert_eq!(
13747            failure_signature("process exited with no usable output (exit code 2)"),
13748            failure_signature("process exited with no usable output (exit code 137)")
13749        );
13750        for (old, new) in [
13751            (
13752                "exited with Some(2) and no usable output",
13753                "process exited with no usable output (exit code 2)",
13754            ),
13755            (
13756                "exited with None and no usable output",
13757                "process exited with no usable output (exit code unavailable)",
13758            ),
13759        ] {
13760            let previous = FailClass::Other(failure_signature(old));
13761            let current = FailClass::Other(failure_signature(new));
13762            assert_eq!(previous, current, "resumption retains the failure class");
13763            assert!(!should_hand_over(Some(&previous), &current));
13764        }
13765        assert_eq!(
13766            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13767            failure_signature("cannot open /var/x.txt: denied")
13768        );
13769        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13770    }
13771
13772    #[test]
13773    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13774        let other = FailClass::Other("x".into());
13775        assert!(should_hand_over(None, &FailClass::Quota));
13776        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13777        assert!(should_hand_over(
13778            Some(&FailClass::Timeout),
13779            &FailClass::Timeout
13780        ));
13781        assert!(should_hand_over(None, &other));
13782        assert!(!should_hand_over(Some(&other), &other));
13783        assert!(should_hand_over(
13784            Some(&other),
13785            &FailClass::Other("y".into())
13786        ));
13787        // A quota or timeout in between ends the run of identical failures.
13788        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13789        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13790    }
13791
13792    #[test]
13793    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13794        let a = SeatState::new("judge-1", "alpha", 7);
13795        let b = handover_seat("judge-1", "beta", 7);
13796        assert_ne!(a.claude_session, b.claude_session);
13797        assert_eq!(b.turns, 0);
13798    }
13799
13800    #[test]
13801    fn a_timeout_is_classified_apart_from_other_failures() {
13802        assert_eq!(
13803            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13804            Some(FailClass::Timeout)
13805        );
13806        assert!(matches!(
13807            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13808            Some(FailClass::Other(_))
13809        ));
13810    }
13811
13812    #[test]
13813    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13814        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13815        let tried = BTreeSet::from(["beta".to_owned()]);
13816        // beta sits at index 1; the next candidate is gamma, never alpha —
13817        // which is very likely a different candidate slot's own agent.
13818        let next = next_untried_in_roster(&roster, 1, &tried);
13819        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13820    }
13821
13822    #[test]
13823    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13824        let roster = vec![spec("alpha"), spec("beta")];
13825        let tried = BTreeSet::from(["beta".to_owned()]);
13826        // beta is the roster's last entry: nothing follows it, and alpha —
13827        // earlier in the roster, almost certainly a different candidate
13828        // slot's own agent — must not be reached by wrapping back to it.
13829        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13830    }
13831
13832    #[test]
13833    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13834        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13835        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13836        // beta (index 1) and gamma (index 2, the only entry after it) have
13837        // both been tried; alpha (index 0) never has, but it comes before
13838        // beta's own position, so there is nothing further for this seat.
13839        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13840    }
13841
13842    #[test]
13843    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13844        let roster = vec![spec("a"), spec("a"), spec("b")];
13845        let tried = BTreeSet::from(["a".to_owned()]);
13846        let next = next_untried_in_roster(&roster, 0, &tried);
13847        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13848    }
13849
13850    #[test]
13851    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13852        let roster = vec![spec("a"), spec("b")];
13853        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13854        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13855    }
13856
13857    #[test]
13858    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13859        let dir = tempfile::tempdir().unwrap();
13860        let bay = dir.path().join("ffff");
13861
13862        // Not there yet: nothing to do, nothing to panic on.
13863        remove_if_empty(&bay);
13864        assert!(!bay.exists());
13865
13866        // Something still inside - the winner's worktree, or a stray file -
13867        // keeps the directory standing.
13868        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13869        remove_if_empty(&bay);
13870        assert!(bay.exists(), "non-empty directory must survive");
13871
13872        // Once the last entry is gone, so is the directory itself.
13873        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13874        remove_if_empty(&bay);
13875        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13876    }
13877
13878    // `round_is_clean` is the exact decision this task fixed: a round with a
13879    // seat that never answered must not read the same as a round every seat
13880    // actually reviewed. These are deterministic and process-free by design —
13881    // the equivalent end-to-end check (a real reviewer timing out under a
13882    // live graph run) is a genuine race against wall-clock contention, and a
13883    // spawn slow enough to blow even a generous budget under a loaded test
13884    // run must not turn this specific regression check flaky.
13885
13886    #[test]
13887    fn a_full_panel_that_found_nothing_is_clean() {
13888        assert!(round_is_clean(
13889            0,
13890            true,
13891            2,
13892            2,
13893            0,
13894            IncompleteReviewPolicy::Block
13895        ));
13896    }
13897
13898    #[test]
13899    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13900        assert!(!round_is_clean(
13901            0,
13902            true,
13903            1,
13904            2,
13905            0,
13906            IncompleteReviewPolicy::Block
13907        ));
13908    }
13909
13910    #[test]
13911    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13912        assert!(!round_is_clean(
13913            1,
13914            true,
13915            1,
13916            2,
13917            0,
13918            IncompleteReviewPolicy::Warn
13919        ));
13920    }
13921
13922    #[test]
13923    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13924        assert!(round_is_clean(
13925            0,
13926            true,
13927            1,
13928            2,
13929            0,
13930            IncompleteReviewPolicy::Warn
13931        ));
13932    }
13933
13934    #[test]
13935    fn a_full_panel_with_an_open_finding_is_not_clean() {
13936        assert!(!round_is_clean(
13937            1,
13938            true,
13939            2,
13940            2,
13941            0,
13942            IncompleteReviewPolicy::Block
13943        ));
13944    }
13945
13946    #[test]
13947    fn a_full_panel_with_a_red_e2e_is_not_clean() {
13948        assert!(!round_is_clean(
13949            0,
13950            false,
13951            2,
13952            2,
13953            0,
13954            IncompleteReviewPolicy::Block
13955        ));
13956    }
13957
13958    // The stall this task closes: under the default `block` policy, a seat
13959    // missing only because it was rate limited must not force a wait for a
13960    // session limit that will not lift by the next round. `round_is_clean`
13961    // is where that quorum carve-out lives; the review loop around it never
13962    // changes what a reviewer's vote or a finding's severity means.
13963
13964    #[test]
13965    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
13966        // 1 of 2 answered, and the one missing was quota'd — the exact
13967        // "review-2 rate limited (quota)" shape from the field report.
13968        assert!(round_is_clean(
13969            0,
13970            true,
13971            1,
13972            2,
13973            1,
13974            IncompleteReviewPolicy::Block
13975        ));
13976    }
13977
13978    #[test]
13979    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
13980        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
13981        // not a quota loss (`quota_missing` stays 0) — worth another try.
13982        assert!(!round_is_clean(
13983            0,
13984            true,
13985            1,
13986            2,
13987            0,
13988            IncompleteReviewPolicy::Block
13989        ));
13990    }
13991
13992    #[test]
13993    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
13994        assert!(!round_is_clean(
13995            1,
13996            true,
13997            1,
13998            2,
13999            1,
14000            IncompleteReviewPolicy::Block
14001        ));
14002        assert!(!round_is_clean(
14003            0,
14004            false,
14005            1,
14006            2,
14007            1,
14008            IncompleteReviewPolicy::Block
14009        ));
14010    }
14011
14012    #[test]
14013    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14014        // Every seat quota'd, nobody answered: there is no panel to decide
14015        // on, so this must fall through to the existing block-and-retry
14016        // fallback rather than call an unreviewed patch clean.
14017        assert!(!round_is_clean(
14018            0,
14019            true,
14020            0,
14021            2,
14022            2,
14023            IncompleteReviewPolicy::Block
14024        ));
14025    }
14026
14027    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14028        CommandOutcome {
14029            command: "test".to_owned(),
14030            code,
14031            output_tail: String::new(),
14032            duration_ms: 0,
14033            resource_blocked,
14034        }
14035    }
14036
14037    #[cfg(unix)]
14038    #[tokio::test]
14039    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14040        ask_test_home();
14041        let temp = tempfile::tempdir().unwrap();
14042        let mut runner = runner_at(RunStatus::Gating);
14043        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14044        let commands = vec![
14045            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14046            "touch must-not-run".to_owned(),
14047        ];
14048        let (outcomes, retried, pids) = run_e2e_with_retry(
14049            &mut runner.state,
14050            &shell,
14051            &commands,
14052            temp.path(),
14053            Duration::from_secs(5),
14054            "fixture",
14055        )
14056        .await;
14057        assert_eq!(outcomes.len(), 1);
14058        assert!(outcomes[0].resource_blocked);
14059        assert!(!outcomes[0].build_failed());
14060        assert!(!retried);
14061        assert!(pids.is_empty());
14062        assert!(!temp.path().join("must-not-run").exists());
14063        assert_eq!(
14064            runner.state.resource_block.as_ref().unwrap().kind,
14065            "incremental cache admission"
14066        );
14067    }
14068
14069    #[test]
14070    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14071        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14072        assert!(
14073            !verify_inconclusive(&[outcome(Some(1), false)]),
14074            "an ordinary failure is still evidence about the patch"
14075        );
14076        assert!(verify_inconclusive(&[outcome(None, true)]));
14077        assert!(
14078            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14079            "one inconclusive outcome taints the whole batch"
14080        );
14081        assert!(!verify_inconclusive(&[]));
14082    }
14083
14084    #[tokio::test]
14085    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14086        // Alive for the first two checks, then dead - confirms the loop
14087        // actually re-polls rather than deciding once and sleeping out the
14088        // ceiling regardless.
14089        let calls = std::sync::atomic::AtomicUsize::new(0);
14090        let started = Instant::now();
14091        wait_for_pids_with(
14092            &[123],
14093            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14094            Duration::from_millis(5),
14095            Duration::from_secs(5),
14096        )
14097        .await;
14098        assert!(
14099            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14100            "must keep checking rather than deciding on the first answer"
14101        );
14102        assert!(
14103            started.elapsed() < Duration::from_secs(1),
14104            "must return the moment it is confirmed dead, not wait out the ceiling"
14105        );
14106    }
14107
14108    #[tokio::test]
14109    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14110        let started = Instant::now();
14111        wait_for_pids_with(
14112            &[123],
14113            |_| true, // never reports dead
14114            Duration::from_millis(5),
14115            Duration::from_millis(30),
14116        )
14117        .await;
14118        let elapsed = started.elapsed();
14119        assert!(
14120            elapsed >= Duration::from_millis(30),
14121            "must not give up before its own ceiling: {elapsed:?}"
14122        );
14123        assert!(
14124            elapsed < Duration::from_secs(1),
14125            "must not wait past its own ceiling either: {elapsed:?}"
14126        );
14127    }
14128
14129    #[tokio::test]
14130    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14131        let started = Instant::now();
14132        wait_for_pids_with(
14133            &[],
14134            |_| true,
14135            Duration::from_secs(5),
14136            Duration::from_secs(5),
14137        )
14138        .await;
14139        assert!(
14140            started.elapsed() < Duration::from_millis(200),
14141            "an empty pid list has nothing to confirm"
14142        );
14143    }
14144
14145    // `review_conclusion` is the exact decision the review hand-off task
14146    // fixed: a round budget spent (or a tree that stopped moving) must not
14147    // collapse into `Blocked` regardless of what verification actually
14148    // said. Deterministic and process-free for the same reason the
14149    // `round_is_clean` family above is.
14150    fn review_round(
14151        clean: bool,
14152        blocking: usize,
14153        answered: usize,
14154        expected: usize,
14155        progressed: bool,
14156        e2e_ok: bool,
14157    ) -> ReviewRound {
14158        ReviewRound {
14159            round: 1,
14160            head: "h".to_owned(),
14161            verified_head: None,
14162            verified_at: None,
14163            reviews: Vec::new(),
14164            e2e: vec![CommandOutcome {
14165                command: "test".to_owned(),
14166                code: Some(if e2e_ok { 0 } else { 1 }),
14167                output_tail: String::new(),
14168                duration_ms: 0,
14169                resource_blocked: false,
14170            }],
14171            verify_retried: false,
14172            e2e_deferred: false,
14173            e2e_defer_reason: None,
14174            fix: None,
14175            blocking,
14176            answered,
14177            expected,
14178            clean,
14179            progressed,
14180            vote_split: false,
14181            reconsideration: Vec::new(),
14182            verdict: None,
14183        }
14184    }
14185
14186    #[test]
14187    fn review_conclusion_is_none_when_nothing_has_run() {
14188        assert_eq!(review_conclusion(&[], 3), None);
14189    }
14190
14191    #[test]
14192    fn review_conclusion_is_none_while_rounds_remain() {
14193        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14194        assert_eq!(review_conclusion(&rounds, 3), None);
14195    }
14196
14197    #[test]
14198    fn review_conclusion_is_gating_once_a_round_is_clean() {
14199        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14200        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14201    }
14202
14203    #[test]
14204    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14205        let rounds = vec![
14206            review_round(false, 1, 2, 2, true, true),
14207            review_round(false, 1, 2, 2, true, true),
14208        ];
14209        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14210    }
14211
14212    #[test]
14213    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14214        let rounds = vec![
14215            review_round(false, 1, 2, 2, true, true),
14216            review_round(false, 1, 2, 2, true, false),
14217        ];
14218        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14219    }
14220
14221    #[test]
14222    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14223        // Magi never got a command to run against this round's own head — a
14224        // resource-blocked attempt, not a red one — so this must never
14225        // settle on `Blocked` the way a genuine e2e failure would. `None`
14226        // here is what tells `Runner::review_loop` to retry the check
14227        // itself rather than trust this cheap recomputation with a verdict
14228        // it cannot actually produce.
14229        let mut blocked = review_round(false, 1, 2, 2, true, false);
14230        blocked.e2e[0].resource_blocked = true;
14231        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14232        assert_eq!(review_conclusion(&rounds, 2), None);
14233    }
14234
14235    #[test]
14236    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14237        // Missing input, not a verified tree — never a hand-off candidate.
14238        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14239        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14240    }
14241
14242    #[test]
14243    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14244        let rounds = vec![
14245            review_round(false, 1, 2, 2, false, true),
14246            review_round(false, 1, 2, 2, false, true),
14247        ];
14248        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14249    }
14250
14251    fn secs(n: u64) -> Duration {
14252        Duration::from_secs(n)
14253    }
14254
14255    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14256    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14257    #[tokio::test]
14258    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14259        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14260        if std::env::var_os(CHILD).is_none() {
14261            // Other tests reconcile every run in the process-global home,
14262            // briefly taking owner locks. This ancestry test expects an idle
14263            // predecessor, so give it its own process and pinned test home.
14264            let output = std::process::Command::new(std::env::current_exe().unwrap())
14265                .args([
14266                    "--exact",
14267                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14268                    "--nocapture",
14269                ])
14270                .env(CHILD, "1")
14271                .env_remove("MAGI_WORKER_HOME")
14272                .env_remove("MAGI_WORKER_ID")
14273                .output()
14274                .unwrap();
14275            assert!(
14276                output.status.success()
14277                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14278                "{}\n{}",
14279                String::from_utf8_lossy(&output.stdout),
14280                String::from_utf8_lossy(&output.stderr)
14281            );
14282            return;
14283        }
14284        ask_test_home();
14285        let tmp = tempfile::tempdir().unwrap();
14286        let repo = tmp.path().join("repo");
14287        std::fs::create_dir_all(&repo).unwrap();
14288        init_repo(&repo);
14289        let g = |args: &[&str]| {
14290            let out = std::process::Command::new("git")
14291                .args(args)
14292                .current_dir(&repo)
14293                .quiet()
14294                .output()
14295                .unwrap();
14296            assert!(
14297                out.status.success(),
14298                "{}",
14299                String::from_utf8_lossy(&out.stderr)
14300            );
14301        };
14302        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14303        g(&["checkout", "-b", "historical"]);
14304        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14305        g(&["add", "."]);
14306        g(&["commit", "-m", "old alternative"]);
14307        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14308        g(&["checkout", "-b", "magi/task-selected", &base]);
14309        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14310        g(&["add", "."]);
14311        g(&["commit", "-m", "measurement infrastructure"]);
14312        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14313        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14314        g(&["add", "."]);
14315        g(&["commit", "-m", "dependent implementation"]);
14316        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14317        g(&["checkout", "main"]);
14318        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14319        g(&["add", "."]);
14320        g(&["commit", "-m", "unrelated base change"]);
14321        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14322        let origin = tmp.path().join("origin.git");
14323        g(&[
14324            "clone",
14325            "--bare",
14326            repo.to_str().unwrap(),
14327            origin.to_str().unwrap(),
14328        ]);
14329        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14330        g(&["fetch", "origin"]);
14331        let selection = crate::pr_recovery::Selection {
14332            url: "https://example.test/project/pull/1".into(),
14333            branch: "magi/task-selected".into(),
14334            head: head.clone(),
14335            source_head: None,
14336            source_run: None,
14337            source_proof: None,
14338            disposition: Some(crate::pr_recovery::Disposition::Review),
14339            instruction: "preserve measured behavior".into(),
14340            approved_at: jiff::Timestamp::now(),
14341        };
14342        let mut template = runner_at(RunStatus::Reviewing);
14343        template.state.repo = repo.clone();
14344        template.state.base_commit = current.clone();
14345        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14346        template.state.origin = Some(crate::run::Origin::queue("selected"));
14347        template.state.pr_head = Some(selection.branch.clone());
14348        template.state.pr_recovery = Some(selection.clone());
14349        let mut runner =
14350            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14351                .await
14352                .unwrap();
14353        let wt = runner.state.winner().unwrap().worktree.clone();
14354        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14355        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14356        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14357        assert!(runner.state.seeds.is_empty());
14358        assert_eq!(runner.state.instruction, "preserve measured behavior");
14359        runner.sync_to_base().await.unwrap();
14360        assert_eq!(
14361            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14362            "prerequisite"
14363        );
14364        assert_eq!(
14365            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14366            "measured implementation"
14367        );
14368        assert_eq!(
14369            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14370            "current main change"
14371        );
14372        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14373        assert_eq!(
14374            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14375            head
14376        );
14377
14378        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14379        let mut partial = runner_at(RunStatus::Reviewing);
14380        partial.state.repo = repo.clone();
14381        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14382        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14383            run: Some("exit 1".into()),
14384            ..Default::default()
14385        }];
14386        let selection = runner.state.pr_recovery.clone().unwrap();
14387        partial.state.pr_recovery = Some(selection.clone());
14388        partial.state.pr_head = Some(selection.branch.clone());
14389        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14390        let id = partial.state.id.clone();
14391        let error = match Runner::open_selected(
14392            &repo,
14393            partial.state,
14394            partial.roles,
14395            1,
14396            runner.state.base_commit.clone(),
14397            selection,
14398        )
14399        .await
14400        {
14401            Ok(_) => panic!("setup must fail"),
14402            Err(e) => e,
14403        };
14404        assert_eq!(
14405            error
14406                .downcast_ref::<crate::run::StartupFailure>()
14407                .unwrap()
14408                .run,
14409            id
14410        );
14411        let failed = RunState::load(&id).unwrap();
14412        assert_eq!(failed.status, RunStatus::Failed);
14413        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14414        assert!(Runner::resume(&id).is_err());
14415        runner.state.status = RunStatus::Ready;
14416        runner.state.driver_exited = true;
14417        runner.state.save().unwrap();
14418        let mut retry = runner_at(RunStatus::Reviewing);
14419        retry.state.repo = repo.clone();
14420        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14421        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14422        reselected.approved_at = jiff::Timestamp::now();
14423        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14424        retry.state.pr_head = Some(reselected.branch.clone());
14425        retry.state.pr_recovery = Some(reselected.clone());
14426        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14427            .await
14428            .unwrap();
14429        let restarted = Runner::open_selected(
14430            &repo,
14431            retry.state,
14432            retry.roles,
14433            1,
14434            runner.state.base_commit.clone(),
14435            reselected,
14436        )
14437        .await
14438        .unwrap();
14439        assert_eq!(
14440            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14441                .await
14442                .unwrap(),
14443            head
14444        );
14445        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14446        assert_eq!(
14447            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14448            head
14449        );
14450    }
14451
14452    #[tokio::test]
14453    #[cfg(unix)]
14454    async fn fallback_and_interrupted_restoration_require_coordination() {
14455        use std::os::fd::AsRawFd as _;
14456        ask_test_home();
14457        let tmp = tempfile::tempdir().unwrap();
14458        let mut runner = registered_runner(tmp.path()).await;
14459        runner.state.config.graph.timeout_resource_wait = 1;
14460        let winner = runner.state.candidates[0].clone();
14461        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14462        let leftover = winner.worktree.join("leftover");
14463        std::fs::write(&leftover, "untrusted edits").unwrap();
14464        let next = tmp.path().join("next-fixer");
14465        let mut failed = spec("first");
14466        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14467        let mut successor = spec("second");
14468        successor.command = vec![
14469            "sh".into(),
14470            "-c".into(),
14471            format!("touch '{}'; echo done", next.display()),
14472        ];
14473        let attempts = vec![
14474            (failed.clone(), "first".into()),
14475            (successor.clone(), "second".into()),
14476        ];
14477        let lock_path = runner.state.repo.join(".git/updater.lock");
14478        let external = std::fs::File::create(&lock_path).unwrap();
14479        assert_eq!(
14480            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14481            0
14482        );
14483        let artifacts = tmp.path().join("artifacts");
14484        let build = |spec, seat| SeatJob {
14485            spec,
14486            seat,
14487            cwd: winner.worktree.clone(),
14488            prompt: "fixture".into(),
14489            timeout: Duration::from_secs(10),
14490            allow_write: true,
14491            sessions: false,
14492            artifacts: artifacts.clone(),
14493            stem: "fix".into(),
14494            handover: None,
14495        };
14496        let (_, _, out) = runner
14497            .ask_fixer_among(
14498                attempts.clone(),
14499                "graft",
14500                None,
14501                Some((&winner.worktree, &head)),
14502                build,
14503            )
14504            .await;
14505        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14506        assert!(leftover.exists(), "contention must prevent clean");
14507        assert!(!next.exists());
14508        assert!(runner.state.handovers.is_empty());
14509        // Early interrupted-repair rollback uses the same protected route.
14510        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14511            from_run: "earlier".into(),
14512            generation: 1,
14513            mode: crate::recovery::Mode::VerifyRepair,
14514            reasons: Vec::new(),
14515            reviewed_head: None,
14516            rounds: Vec::new(),
14517            open_findings: Vec::new(),
14518            evidence: None,
14519            criteria_inherited: true,
14520            repair: crate::recovery::Repair::Started {
14521                before: head.clone(),
14522            },
14523        });
14524        assert!(
14525            runner
14526                .recovery_repair(&winner)
14527                .await
14528                .unwrap_err()
14529                .is::<crate::repo_lock::ResourceBlocked>()
14530        );
14531        assert!(matches!(
14532            runner.state.recovery.as_ref().unwrap().repair,
14533            crate::recovery::Repair::Started { .. }
14534        ));
14535        assert!(leftover.exists());
14536        drop(external);
14537        runner.recovery_repair(&winner).await.unwrap();
14538        assert!(!leftover.exists());
14539        assert!(matches!(
14540            runner.state.recovery.as_ref().unwrap().repair,
14541            crate::recovery::Repair::Done { .. }
14542        ));
14543        // A hard lock-open failure also leaves HEAD and untracked work intact.
14544        std::fs::remove_file(&lock_path).unwrap();
14545        std::fs::create_dir(&lock_path).unwrap();
14546        std::fs::write(&leftover, "untrusted edits").unwrap();
14547        let (_, _, out) = runner
14548            .ask_fixer_among(
14549                attempts,
14550                "graft",
14551                None,
14552                Some((&winner.worktree, &head)),
14553                build,
14554            )
14555            .await;
14556        assert!(matches!(out, AgentOutcome::Failed(_)));
14557        assert!(leftover.exists());
14558        assert!(!next.exists());
14559        assert_eq!(
14560            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14561            head
14562        );
14563    }
14564
14565    #[cfg(unix)]
14566    async fn registered_runner(root: &Path) -> Runner {
14567        repository_runner(root, crate::config::test_coordination()).await
14568    }
14569
14570    async fn repository_runner(
14571        root: &Path,
14572        coordination: Option<crate::config::Coordination>,
14573    ) -> Runner {
14574        ask_test_home();
14575        let repo = root.join("repo");
14576        std::fs::create_dir_all(&repo).unwrap();
14577        init_repo(&repo);
14578        if coordination.is_some() {
14579            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14580        }
14581        let mut runner = runner_at(RunStatus::Prep);
14582        runner.state.repo = repo.clone();
14583        runner.state.config.coordination = coordination;
14584        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14585        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14586        runner.state.config.graph.advise = false;
14587        runner.roles.implementers = vec![spec("implement")];
14588        runner.roles.judges = vec![spec("judge")];
14589        runner.prep().await.unwrap();
14590        runner
14591    }
14592
14593    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14594
14595    #[tokio::test]
14596    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14597        let tmp = tempfile::tempdir().unwrap();
14598        let mut runner = repository_runner(tmp.path(), None).await;
14599        let prompts = runner.state.config.prompts.clone();
14600        let run = runner.state.id.clone();
14601        let ctx = WaveCtx {
14602            carry_seats: false,
14603            pauses: None,
14604            run: &run,
14605            node: "advise",
14606            prompts: &prompts,
14607            cache: None,
14608            round: None,
14609        };
14610        let jobs: Vec<_> = ["alpha", "beta"]
14611            .into_iter()
14612            .map(|id| SeatJob {
14613                spec: scripted(id, AUTH_OUTAGE),
14614                seat: SeatState::new(id, id, 7),
14615                prompt: "propose".into(),
14616                cwd: runner.state.candidates[0].worktree.clone(),
14617                timeout: Duration::from_secs(10),
14618                allow_write: false,
14619                sessions: false,
14620                artifacts: agent::artifacts_dir(&runner.state.dir()),
14621                stem: id.into(),
14622                handover: None,
14623            })
14624            .collect();
14625        let mut losses = Vec::new();
14626        let results = ask_wave_with(
14627            jobs.clone(),
14628            runner.sem.clone(),
14629            2,
14630            &[],
14631            &ctx,
14632            &mut losses,
14633            &mut runner.state,
14634            &|_: &str| Ok(()),
14635        )
14636        .await;
14637        assert!(
14638            results
14639                .iter()
14640                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14641        );
14642        assert_eq!(runner.state.quota.len(), 2);
14643        assert!(
14644            runner
14645                .state
14646                .quota
14647                .iter()
14648                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14649        );
14650        assert!(!runner.state.advise_attempted);
14651        runner.stop_for_resource().unwrap();
14652        assert_eq!(runner.state.status, RunStatus::Blocked);
14653        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14654        assert_eq!(
14655            runner.state.resource_block.as_ref().unwrap().kind,
14656            "agent authentication"
14657        );
14658        // A machine loss followed by an ordinary failure is not evidence
14659        // that the whole node was unavailable, and must not earn a refund.
14660        runner.state.resource_block = None;
14661        runner.state.quota.clear();
14662        let mut mixed = jobs;
14663        mixed[1].spec.command = vec!["false".into()];
14664        let results = ask_wave_with(
14665            mixed,
14666            runner.sem.clone(),
14667            0,
14668            &[],
14669            &ctx,
14670            &mut losses,
14671            &mut runner.state,
14672            &|_: &str| Ok(()),
14673        )
14674        .await;
14675        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14676        assert!(runner.state.resource_block.is_none());
14677        assert_eq!(losses.len(), 1);
14678    }
14679
14680    #[tokio::test]
14681    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14682        let tmp = tempfile::tempdir().unwrap();
14683        let mut runner = repository_runner(tmp.path(), None).await;
14684        let implementer = scripted("implement", AUTH_OUTAGE);
14685        runner.roles.implementers = vec![implementer.clone()];
14686        runner.state.config.agents = vec![implementer];
14687        runner.implement().await.unwrap();
14688        assert_eq!(runner.state.status, RunStatus::Blocked);
14689        assert_eq!(runner.state.quota.len(), 1);
14690        assert_eq!(
14691            runner.state.quota[0].cause,
14692            agent::QuotaCause::Authentication
14693        );
14694        let candidate = &runner.state.candidates[0];
14695        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14696        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14697    }
14698
14699    #[tokio::test]
14700    #[cfg(unix)]
14701    async fn parked_implementer_persists_completed_work() {
14702        for registered in [false, true] {
14703            let tmp = tempfile::tempdir().unwrap();
14704            let mut runner = registered_runner(tmp.path()).await;
14705            if !registered {
14706                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14707            }
14708            let marker = tmp.path().join("started");
14709            let release = tmp.path().join("release");
14710            let mut implementer = spec("implement");
14711            implementer.command = vec![
14712                "sh".into(),
14713                "-c".into(),
14714                format!(
14715                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14716                    marker.display(),
14717                    release.display()
14718                ),
14719            ];
14720            runner.roles.implementers = vec![implementer.clone()];
14721            runner.state.config.agents = vec![implementer];
14722            let pause = runner.pause.clone();
14723            let (result, ()) = tokio::join!(runner.implement(), async {
14724                tokio::time::timeout(Duration::from_secs(10), async {
14725                    while !marker.exists() {
14726                        tokio::time::sleep(Duration::from_millis(10)).await;
14727                    }
14728                })
14729                .await
14730                .unwrap();
14731                pause.park();
14732                std::fs::write(&release, "go").unwrap();
14733            });
14734            result.unwrap();
14735            let candidate = &runner.state.candidates[0];
14736            assert!(candidate.commits > 0, "{candidate:?}");
14737            assert!(candidate.summary.contains("saved-result"));
14738            assert_eq!(candidate.files, 1);
14739            let before = candidate.commits;
14740            runner.implement().await.unwrap();
14741            assert_eq!(runner.state.candidates[0].commits, before);
14742        }
14743    }
14744
14745    #[tokio::test]
14746    #[cfg(unix)]
14747    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14748        let tmp = tempfile::tempdir().unwrap();
14749        let mut runner = registered_runner(tmp.path()).await;
14750        let winner = runner.state.candidates[0].clone();
14751        let cache = tmp.path().join("cache");
14752        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14753        runner.state.config.graph.timeout_resource_wait = 0;
14754        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14755        let crate::cache::AcquireOutcome::Acquired(held) =
14756            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14757        else {
14758            panic!("cache must be free")
14759        };
14760        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14761            from_run: "earlier".into(),
14762            generation: 1,
14763            mode: crate::recovery::Mode::VerifyRepair,
14764            reasons: Vec::new(),
14765            reviewed_head: None,
14766            rounds: Vec::new(),
14767            open_findings: Vec::new(),
14768            evidence: None,
14769            criteria_inherited: true,
14770            repair: crate::recovery::Repair::NotStarted,
14771        });
14772        runner.recovery_repair(&winner).await.unwrap();
14773        assert!(matches!(
14774            runner.state.recovery.as_ref().unwrap().repair,
14775            crate::recovery::Repair::WaitingForRestore { .. }
14776        ));
14777        drop(held);
14778        runner.state.resource_block = None;
14779        runner.state.config.graph.timeout_resource_wait = 1;
14780        runner.recovery_repair(&winner).await.unwrap();
14781        assert!(matches!(
14782            runner.state.recovery.as_ref().unwrap().repair,
14783            crate::recovery::Repair::Done { .. }
14784        ));
14785        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14786    }
14787
14788    #[tokio::test]
14789    #[cfg(unix)]
14790    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14791        use std::os::fd::AsRawFd as _;
14792        for graft in [false, true] {
14793            let tmp = tempfile::tempdir().unwrap();
14794            let mut runner = registered_runner(tmp.path()).await;
14795            runner.state.config.graph.timeout_resource_wait = 1;
14796            let winner = runner.state.candidates[0].clone();
14797            let marker = tmp.path().join("fixer-started");
14798            let mut fixer = spec("fixer");
14799            fixer.command = vec![
14800                "sh".into(),
14801                "-c".into(),
14802                format!("echo started >> '{}'; echo done", marker.display()),
14803            ];
14804            runner.roles.fixer = Some(vec![fixer]);
14805            if graft {
14806                let mut loser = winner.clone();
14807                loser.label = 'B';
14808                loser.index = 1;
14809                runner.state.candidates.push(loser);
14810                runner.state.config.graph.graft_runner_up = true;
14811                runner.state.tally = Some(Tally {
14812                    first_choice: BTreeMap::new(),
14813                    borda: BTreeMap::new(),
14814                    winner: winner.label,
14815                    rankings: 1,
14816                    unanimous_initial: true,
14817                    deliberated: false,
14818                    changed_votes: 0,
14819                    unanimous_final: true,
14820                    tie_break: None,
14821                    judges: 1,
14822                    present: 1,
14823                    quorum: 1,
14824                    met_quorum: true,
14825                    uncontested: None,
14826                });
14827                runner.state.judgements.push(crate::run::Judgement {
14828                    judge: 1,
14829                    seat: "judge-1".into(),
14830                    agent: "judge".into(),
14831                    ranking: vec![winner.label, 'B'],
14832                    reasons: BTreeMap::new(),
14833                    confidence: None,
14834                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14835                    order: vec![0, 1],
14836                    failed: None,
14837                    duration_ms: 0,
14838                });
14839            } else {
14840                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14841                    from_run: "earlier".into(),
14842                    generation: 1,
14843                    mode: crate::recovery::Mode::VerifyRepair,
14844                    reasons: Vec::new(),
14845                    reviewed_head: None,
14846                    rounds: Vec::new(),
14847                    open_findings: Vec::new(),
14848                    evidence: None,
14849                    criteria_inherited: true,
14850                    repair: crate::recovery::Repair::NotStarted,
14851                });
14852            }
14853            let reader = crate::repo_lock::SeatLease::for_checkout(
14854                &winner.worktree,
14855                false,
14856                crate::config::test_coordination().as_ref(),
14857            )
14858            .await
14859            .unwrap();
14860            let pause = runner.pause.clone();
14861            let (result, ()) = tokio::join!(
14862                async {
14863                    if graft {
14864                        runner.graft().await
14865                    } else {
14866                        runner.recovery_repair(&winner).await
14867                    }
14868                },
14869                async {
14870                    tokio::time::sleep(Duration::from_millis(100)).await;
14871                    pause.park();
14872                }
14873            );
14874            result.unwrap();
14875            assert!(!marker.exists(), "a blocked seat must not spawn");
14876            // Exercise the persisted state, as a binary replacement/resume does.
14877            runner.state = RunState::load(&runner.state.id).unwrap();
14878            if graft {
14879                assert_eq!(
14880                    runner.state.graft.as_ref().unwrap().outcome,
14881                    GraftOutcome::WaitingForRestore
14882                );
14883            } else {
14884                assert!(matches!(
14885                    runner.state.recovery.as_ref().unwrap().repair,
14886                    crate::recovery::Repair::WaitingForRestore { .. }
14887                ));
14888            }
14889            drop(reader);
14890            runner.pause = Pause::new();
14891            runner.state.resource_block = None;
14892            // Deferred restoration must still respect an external lock owner.
14893            let external =
14894                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14895            assert_eq!(
14896                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14897                0
14898            );
14899            let leftover = winner.worktree.join("leftover");
14900            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14901            let result = if graft {
14902                runner.graft().await
14903            } else {
14904                runner.recovery_repair(&winner).await
14905            };
14906            assert!(
14907                result
14908                    .unwrap_err()
14909                    .is::<crate::repo_lock::ResourceBlocked>()
14910            );
14911            assert!(leftover.exists());
14912            assert!(!marker.exists());
14913            drop(external);
14914            runner.state.resource_block = None;
14915            let result = if graft {
14916                runner.graft().await
14917            } else {
14918                runner.recovery_repair(&winner).await
14919            };
14920            result.unwrap();
14921            assert!(!leftover.exists(), "restore must precede retry");
14922            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14923            if graft {
14924                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14925            } else {
14926                assert!(matches!(
14927                    runner.state.recovery.as_ref().unwrap().repair,
14928                    crate::recovery::Repair::Done { .. }
14929                ));
14930            }
14931        }
14932    }
14933
14934    #[tokio::test]
14935    #[cfg(unix)]
14936    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14937        use std::os::fd::AsRawFd as _;
14938        let tmp = tempfile::tempdir().unwrap();
14939        let mut runner = registered_runner(tmp.path()).await;
14940        runner.state.config.graph.timeout_resource_wait = 1;
14941        let winner = runner.state.candidates[0].clone();
14942        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14943        runner.state.tally = Some(Tally {
14944            first_choice: BTreeMap::new(),
14945            borda: BTreeMap::new(),
14946            winner: winner.label,
14947            rankings: 0,
14948            unanimous_initial: false,
14949            deliberated: false,
14950            changed_votes: 0,
14951            unanimous_final: false,
14952            tie_break: None,
14953            judges: 0,
14954            present: 0,
14955            quorum: 0,
14956            met_quorum: true,
14957            uncontested: Some("fixture".into()),
14958        });
14959        runner.state.graft = Some(GraftRecord {
14960            pre: pre.clone(),
14961            post: None,
14962            lines: None,
14963            limit: None,
14964            agent: None,
14965            outcome: GraftOutcome::Running,
14966            requested: Vec::new(),
14967            adoption: None,
14968            adoption_error: None,
14969        });
14970        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
14971        git::commit_all(&winner.worktree, "unfinished graft")
14972            .await
14973            .unwrap();
14974        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14975        let leftover = winner.worktree.join("leftover");
14976        std::fs::write(&leftover, "untrusted edits").unwrap();
14977        let path = runner.state.repo.join(".git/updater.lock");
14978        let external = std::fs::File::create(&path).unwrap();
14979        assert_eq!(
14980            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14981            0
14982        );
14983        assert!(
14984            runner
14985                .graft()
14986                .await
14987                .unwrap_err()
14988                .is::<crate::repo_lock::ResourceBlocked>()
14989        );
14990        assert!(matches!(
14991            runner.state.graft.as_ref().unwrap().outcome,
14992            GraftOutcome::Running
14993        ));
14994        assert_eq!(
14995            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14996            unfinished
14997        );
14998        assert!(leftover.exists());
14999        drop(external);
15000        std::fs::remove_file(&path).unwrap();
15001        std::fs::create_dir(&path).unwrap();
15002        assert!(runner.graft().await.is_err());
15003        assert!(matches!(
15004            runner.state.graft.as_ref().unwrap().outcome,
15005            GraftOutcome::Running
15006        ));
15007        assert_eq!(
15008            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15009            unfinished
15010        );
15011        assert!(leftover.exists());
15012        std::fs::remove_dir(&path).unwrap();
15013        runner.graft().await.unwrap();
15014        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15015        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15016        assert!(!leftover.exists());
15017    }
15018
15019    #[tokio::test]
15020    #[cfg(unix)]
15021    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15022        use std::os::fd::AsRawFd as _;
15023        let tmp = tempfile::tempdir().unwrap();
15024        let mut runner = registered_runner(tmp.path()).await;
15025        runner.state.prep_complete = Some(false);
15026        runner.state.config.graph.timeout_resource_wait = 1;
15027        runner.state.save().unwrap();
15028        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15029        assert_eq!(
15030            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15031            0
15032        );
15033        runner.execute().await.unwrap();
15034        let disk = RunState::load(&runner.state.id).unwrap();
15035        assert!(disk.driver_exited);
15036        assert_eq!(disk.driver_pid, None);
15037        assert_eq!(runner.state.status, RunStatus::Blocked);
15038        assert_eq!(
15039            runner.state.resource_block.as_ref().unwrap().kind,
15040            "repository coordination"
15041        );
15042        assert!(runner.state.prep_failure.is_none());
15043        assert!(runner.state.seats.is_empty());
15044        runner.pause.park();
15045        // Include headroom for Git subprocesses under parallel test load;
15046        // the repository contention deadline itself remains one second.
15047        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15048            .await
15049            .unwrap()
15050            .unwrap();
15051        assert!(runner.state.parked);
15052        assert!(runner.state.prep_failure.is_none());
15053        let disk = RunState::load(&runner.state.id).unwrap();
15054        assert!(disk.driver_exited);
15055        assert_eq!(disk.driver_pid, None);
15056    }
15057
15058    #[tokio::test]
15059    #[cfg(unix)]
15060    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15061        use std::os::fd::AsRawFd as _;
15062        let tmp = tempfile::tempdir().unwrap();
15063        let mut runner = registered_runner(tmp.path()).await;
15064        runner.state.prep_complete = Some(false);
15065        runner.state.config.graph.timeout_resource_wait = 30;
15066        runner.state.save().unwrap();
15067        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15068        assert_eq!(
15069            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15070            0
15071        );
15072        assert!(
15073            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15074                .await
15075                .is_err()
15076        );
15077        let disk = RunState::load(&runner.state.id).unwrap();
15078        assert!(disk.driver_exited);
15079        assert_eq!(disk.driver_pid, None);
15080        assert!(disk.status.resumable());
15081        assert!(runner.state.driver_exited);
15082        assert_eq!(runner.state.driver_pid, None);
15083        runner.state.save().unwrap();
15084        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15085    }
15086
15087    #[tokio::test]
15088    #[cfg(unix)]
15089    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15090        let tmp = tempfile::tempdir().unwrap();
15091        let mut runner = registered_runner(tmp.path()).await;
15092        let winner = runner.state.candidates[0].clone();
15093        let cache = tmp.path().join("cache");
15094        let started = tmp.path().join("pre-gate-started");
15095        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15096        runner.state.config.verify.gate =
15097            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15098        let home = crate::run::home();
15099        let owner =
15100            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15101        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15102            crate::cache::AcquireOutcome::Acquired(g) => g,
15103            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15104        };
15105        let mut gate = Box::pin(runner.run_gate(&winner));
15106        assert!(
15107            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15108                .await
15109                .is_err()
15110        );
15111        assert!(!started.exists(), "pre-gate must wait for the cache too");
15112        // Simulate the wave that already owns the cache entering invoke.
15113        // It must obtain the editor reservation and finish without waiting
15114        // for the gate that is queued behind its cache.
15115        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15116        let editor = crate::repo_lock::SeatLease::checkout_with(
15117            &winner.worktree,
15118            true,
15119            crate::config::test_coordination().as_ref(),
15120            &wait,
15121        )
15122        .await
15123        .unwrap();
15124        drop(editor);
15125        guard.release();
15126        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15127            .await
15128            .unwrap()
15129            .unwrap();
15130        drop(gate);
15131        assert!(outcomes.iter().all(CommandOutcome::ok));
15132        assert!(started.exists());
15133    }
15134
15135    #[tokio::test]
15136    #[cfg(unix)]
15137    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15138        use std::os::fd::AsRawFd as _;
15139
15140        ask_test_home();
15141        let tmp = tempfile::tempdir().unwrap();
15142        let repo = tmp.path().join("repo");
15143        std::fs::create_dir_all(&repo).unwrap();
15144        init_repo(&repo);
15145        let common = repo.join(".git");
15146        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15147        let mut runner = runner_at(RunStatus::Prep);
15148        runner.state.repo = repo.clone();
15149        runner.state.config.coordination = crate::config::test_coordination();
15150        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15151        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15152        runner.state.config.graph.advise = false;
15153        runner.roles.implementers = vec![spec("implement")];
15154        runner.roles.judges = vec![spec("judge")];
15155        runner.prep().await.unwrap();
15156        let winner = runner.state.candidates[0].clone();
15157        // Emulate edits already left by a completed seat. The fixture CLI
15158        // only reports completion; it performs no repository operations.
15159        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15160        let completed = tmp.path().join("seat-completed");
15161        runner.roles.implementers[0].command = vec![
15162            "sh".into(),
15163            "-c".into(),
15164            format!("touch '{}'; echo completed", completed.display()),
15165        ];
15166        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15167        assert_eq!(
15168            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15169            0
15170        );
15171        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15172        let mut implement = Box::pin(runner.implement());
15173        tokio::select! {
15174            result = &mut implement => panic!("rescue must wait: {result:?}"),
15175            result = tokio::time::timeout(Duration::from_secs(10), async {
15176                while !completed.exists() {
15177                    tokio::time::sleep(Duration::from_millis(20)).await;
15178                }
15179            }) => result.unwrap(),
15180        }
15181        assert!(
15182            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15183                .await
15184                .is_err()
15185        );
15186        assert_eq!(
15187            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15188            before
15189        );
15190        assert_eq!(
15191            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15192            0
15193        );
15194        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15195            .await
15196            .unwrap()
15197            .unwrap();
15198        drop(implement);
15199        assert_ne!(
15200            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15201            before
15202        );
15203        assert!(git::is_clean(&winner.worktree).await.unwrap());
15204
15205        let started = tmp.path().join("pre-gate-started");
15206        let proceed = tmp.path().join("pre-gate-proceed");
15207        let checked = tmp.path().join("gate-ran");
15208        runner.state.config.verify.pre_gate = vec![format!(
15209            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15210            started.display(),
15211            proceed.display()
15212        )];
15213        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15214        assert_eq!(
15215            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15216            0
15217        );
15218        let mut gate = Box::pin(runner.run_gate(&winner));
15219        assert!(
15220            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15221                .await
15222                .is_err()
15223        );
15224        assert!(!started.exists());
15225        assert!(!checked.exists());
15226        assert!(git::is_clean(&winner.worktree).await.unwrap());
15227        assert_eq!(
15228            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15229            0
15230        );
15231        tokio::select! {
15232            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15233            result = tokio::time::timeout(Duration::from_secs(10), async {
15234                while !started.exists() {
15235                    tokio::time::sleep(Duration::from_millis(20)).await;
15236                }
15237            }) => result.unwrap(),
15238        }
15239        assert_eq!(
15240            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15241            -1,
15242            "mechanical edits must hold the real lock"
15243        );
15244        std::fs::write(proceed, "ready").unwrap();
15245        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15246            .await
15247            .unwrap()
15248            .unwrap();
15249        drop(gate);
15250        assert!(outcomes.iter().all(CommandOutcome::ok));
15251        assert!(checked.exists());
15252        assert!(runner.state.pre_gate_commit.is_some());
15253        assert!(git::is_clean(&winner.worktree).await.unwrap());
15254        assert_eq!(
15255            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15256            0,
15257            "gate completion must release coordination"
15258        );
15259        drop(outsider);
15260        std::fs::remove_file(common.join("updater.lock")).unwrap();
15261        std::fs::create_dir(common.join("updater.lock")).unwrap();
15262        std::fs::remove_file(&started).unwrap();
15263        std::fs::remove_file(&checked).unwrap();
15264        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15265        let error = runner.run_gate(&winner).await.unwrap_err();
15266        assert!(
15267            error
15268                .to_string()
15269                .contains("acquire mandatory repository lock")
15270        );
15271        assert!(
15272            !started.exists(),
15273            "failed acquisition must prevent pre-gate"
15274        );
15275        assert!(!checked.exists(), "failed acquisition must prevent gate");
15276        assert!(git::is_clean(&winner.worktree).await.unwrap());
15277        assert_eq!(
15278            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15279            head
15280        );
15281    }
15282
15283    #[tokio::test]
15284    #[cfg(unix)]
15285    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15286        use std::os::fd::AsRawFd as _;
15287        use std::time::Duration;
15288
15289        ask_test_home();
15290        let tmp = tempfile::tempdir().unwrap();
15291        let repo = tmp.path().join("repo");
15292        std::fs::create_dir_all(&repo).unwrap();
15293        init_repo(&repo);
15294        let common = repo.join(".git");
15295        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15296        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15297        assert_eq!(
15298            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15299            0
15300        );
15301        let mut runner = runner_at(RunStatus::Prep);
15302        runner.state.repo = repo.clone();
15303        runner.state.config.coordination = crate::config::test_coordination();
15304        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15305        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15306        runner.state.config.graph.advise = true;
15307        runner.state.config.graph.advisors = 1;
15308        runner.roles.implementers = vec![spec("implement")];
15309        runner.roles.judges = vec![spec("judge")];
15310        let root = runner.state.worktree_root();
15311        let started = tmp.path().join("setup-started");
15312        let proceed = tmp.path().join("setup-proceed");
15313        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15314            run: Some(format!(
15315                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15316                started.display(),
15317                proceed.display()
15318            )),
15319            ..Default::default()
15320        }];
15321        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15322        let config = git::git(&repo, &["config", "--local", "--list"])
15323            .await
15324            .unwrap();
15325        let mut prep = Box::pin(runner.prep());
15326        assert!(
15327            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15328                .await
15329                .is_err()
15330        );
15331        assert!(
15332            !root.exists(),
15333            "contention must prevent even checkout creation"
15334        );
15335        assert!(!started.exists(), "contention must prevent setup");
15336        assert_eq!(
15337            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15338            branches
15339        );
15340        assert_eq!(
15341            git::git(&repo, &["config", "--local", "--list"])
15342                .await
15343                .unwrap(),
15344            config
15345        );
15346        drop(lock);
15347        tokio::select! {
15348            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15349            result = tokio::time::timeout(Duration::from_secs(10), async {
15350                while !started.exists() {
15351                    tokio::time::sleep(Duration::from_millis(20)).await;
15352                }
15353            }) => result.unwrap(),
15354        }
15355        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15356        assert_eq!(
15357            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15358            -1
15359        );
15360        assert!(
15361            tokio::time::timeout(
15362                Duration::from_millis(100),
15363                crate::repo_lock::SeatLease::for_checkout(
15364                    &repo,
15365                    true,
15366                    crate::config::test_coordination().as_ref()
15367                )
15368            )
15369            .await
15370            .is_err(),
15371            "preparation must also exclude local editors"
15372        );
15373        std::fs::write(proceed, "ready").unwrap();
15374        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15375            .await
15376            .unwrap()
15377            .unwrap();
15378        drop(prep);
15379        assert_eq!(runner.state.prep_complete, Some(true));
15380        assert!(root.join("judge-1").exists());
15381        assert!(root.join("advisor-1").exists());
15382        assert_eq!(
15383            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15384            0
15385        );
15386    }
15387
15388    #[tokio::test]
15389    #[cfg(unix)]
15390    async fn preparation_lock_failure_prevents_repository_changes() {
15391        ask_test_home();
15392        let tmp = tempfile::tempdir().unwrap();
15393        let repo = tmp.path().join("repo");
15394        std::fs::create_dir_all(&repo).unwrap();
15395        init_repo(&repo);
15396        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15397        // A directory at the required lock path makes opening the real lock
15398        // fail, rather than merely exercising an invalid registration.
15399        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15400        let mut runner = runner_at(RunStatus::Prep);
15401        runner.state.repo = repo.clone();
15402        runner.state.config.coordination = crate::config::test_coordination();
15403        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15404        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15405        runner.roles.implementers = vec![spec("implement")];
15406        let error = runner.prep().await.unwrap_err();
15407        assert!(
15408            error
15409                .to_string()
15410                .contains("acquire mandatory repository lock")
15411        );
15412        assert!(!runner.state.worktree_root().exists());
15413        assert!(runner.state.candidates.is_empty());
15414        assert!(!repo.join(".git/config.worktree").exists());
15415        assert_eq!(
15416            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15417            "* main"
15418        );
15419    }
15420
15421    #[tokio::test]
15422    async fn preparation_records_creation_intent_before_git_can_fail() {
15423        ask_test_home();
15424        let tmp = tempfile::tempdir().unwrap();
15425        let repo = tmp.path().join("repo");
15426        std::fs::create_dir_all(&repo).unwrap();
15427        init_repo(&repo);
15428        let mut runner = runner_at(RunStatus::Prep);
15429        runner.state.repo = repo.clone();
15430        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15431        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15432        runner.state.config.blind.commit_msg_hook = false;
15433        runner.roles.implementers = vec![spec("one")];
15434        let label = blind::assign_labels(1, runner.state.seed)[0];
15435        let branch = runner.state.branch_for(label);
15436        // A conflicting branch makes Git refuse creation before a candidate
15437        // can be registered. The durable intent must still identify it.
15438        git::git(&repo, &["branch", &branch]).await.unwrap();
15439        assert!(runner.prep().await.is_err());
15440        let saved = RunState::load(&runner.state.id).unwrap();
15441        assert!(saved.candidates.is_empty());
15442        let event = saved
15443            .events
15444            .iter()
15445            .find(|e| e.node == "prep-intent")
15446            .unwrap();
15447        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15448        assert_eq!(intent["branch"], branch);
15449        assert_eq!(intent["start"], runner.state.base_commit);
15450        assert_eq!(
15451            intent["worktree"],
15452            runner
15453                .state
15454                .worktree_root()
15455                .join(format!("cand-{label}"))
15456                .to_string_lossy()
15457                .as_ref()
15458        );
15459        assert_eq!(intent["picks"], serde_json::json!([]));
15460    }
15461
15462    #[tokio::test]
15463    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15464        ask_test_home();
15465        let tmp = tempfile::tempdir().unwrap();
15466        let repo = tmp.path().join("repo");
15467        std::fs::create_dir_all(&repo).unwrap();
15468        init_repo(&repo);
15469        let mut runner = runner_at(RunStatus::Prep);
15470        runner.state.repo = repo.clone();
15471        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15472        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15473        runner.state.sources.pick_commits.push(refs::Input {
15474            token: "missing".into(),
15475            sha: "0".repeat(40),
15476            provenance: refs::Provenance::ExplicitCli,
15477        });
15478        assert!(runner.prep().await.is_err());
15479        assert!(runner.state.candidates.is_empty());
15480        assert!(!runner.state.worktree_root().exists());
15481    }
15482
15483    #[tokio::test]
15484    async fn prose_and_legacy_inference_never_import_into_candidates() {
15485        ask_test_home();
15486        let tmp = tempfile::tempdir().unwrap();
15487        let repo = tmp.path().join("repo");
15488        std::fs::create_dir_all(&repo).unwrap();
15489        init_repo(&repo);
15490        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15491        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15492            .await
15493            .unwrap();
15494        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15495        git::git(&repo, &["add", "."]).await.unwrap();
15496        git::git(&repo, &["commit", "-m", "historical implementation"])
15497            .await
15498            .unwrap();
15499        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15500        git::git(&repo, &["checkout", "main"]).await.unwrap();
15501        for instruction in [
15502            format!("Example: magi/history/A at {source}"),
15503            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15504            format!("Never import magi/history/A or {source}"),
15505            format!("Conversation answer: I mentioned {source} only as background"),
15506        ] {
15507            let mut runner = runner_at(RunStatus::Prep);
15508            runner.state.repo = repo.clone();
15509            runner.state.base_commit = base.clone();
15510            runner.state.instruction = instruction.clone();
15511            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15512            runner.state.config.blind.commit_msg_hook = false;
15513            runner.state.seeds = vec![refs::Seed {
15514                token: "magi/history/A".into(),
15515                sha: source.clone(),
15516                kind: refs::SeedKind::Unmerged,
15517                branch: true,
15518                detail: String::new(),
15519            }];
15520            runner.roles.implementers = vec![spec("one")];
15521            assert_eq!(seeded_instruction(&runner.state), instruction);
15522            // Model an old record with inferred seeds but no structured input field.
15523            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15524            legacy.as_object_mut().unwrap().remove("sources");
15525            runner.state = serde_json::from_value(legacy).unwrap();
15526            assert_eq!(runner.state.sources, refs::Sources::default());
15527            runner.state.save().unwrap();
15528            runner.state = RunState::load(&runner.state.id).unwrap();
15529            runner.prep().await.unwrap();
15530            for candidate in &runner.state.candidates {
15531                assert_eq!(
15532                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15533                    base
15534                );
15535                assert!(!candidate.worktree.join("unwanted").exists());
15536            }
15537        }
15538    }
15539
15540    #[tokio::test]
15541    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15542        ask_test_home();
15543        let tmp = tempfile::tempdir().unwrap();
15544        let repo = tmp.path().join("repo");
15545        std::fs::create_dir_all(&repo).unwrap();
15546        init_repo(&repo);
15547        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15548        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15549        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15550        git::git(&repo, &["add", "."]).await.unwrap();
15551        git::git(&repo, &["commit", "-m", "required seed"])
15552            .await
15553            .unwrap();
15554        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15555        git::git(&repo, &["checkout", "main"]).await.unwrap();
15556        let mut runner = runner_at(RunStatus::Prep);
15557        runner.state.repo = repo.clone();
15558        runner.state.base_commit = base.clone();
15559        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15560        runner.state.config.graph.advise = false;
15561        runner.state.config.blind.commit_msg_hook = false;
15562        runner.state.sources =
15563            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15564                .await
15565                .unwrap();
15566        runner.roles.implementers = vec![spec("one"), spec("two")];
15567        runner.roles.judges = vec![spec("judge")];
15568        let label = blind::assign_labels(2, runner.state.seed)[0];
15569        let branch = runner.state.branch_for(label);
15570        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15571        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15572            .await
15573            .unwrap();
15574        runner.state.candidates.push(Candidate {
15575            index: 0,
15576            label,
15577            agent: "one".into(),
15578            branch,
15579            worktree,
15580            summary: String::new(),
15581            stat: String::new(),
15582            files: 0,
15583            commits: 0,
15584            empty: false,
15585            failed: None,
15586            verified_noop: None,
15587            duration_ms: 0,
15588            folded: false,
15589        });
15590        runner.state.save().unwrap();
15591        // Reload the checkpoint immediately before its first cherry-pick.
15592        runner.state = RunState::load(&runner.state.id).unwrap();
15593        assert_eq!(runner.state.prep_complete, Some(false));
15594        runner.prep().await.unwrap();
15595        assert_eq!(runner.state.prep_complete, Some(true));
15596        assert_eq!(runner.state.candidates.len(), 2);
15597        for candidate in &runner.state.candidates {
15598            assert_eq!(
15599                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15600                git::tree_of(&repo, &seed).await.unwrap()
15601            );
15602            assert_eq!(
15603                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15604                "required commit"
15605            );
15606        }
15607        assert!(runner.state.worktree_root().join("judge-1").exists());
15608        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15609        runner.prep().await.unwrap();
15610        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15611
15612        runner.state.prep_complete = Some(false);
15613        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15614            run: Some(format!(
15615                "test -f '{}'",
15616                tmp.path().join("dependency-ready").display()
15617            )),
15618            ..Default::default()
15619        }];
15620        assert!(runner.prep().await.is_err());
15621        assert!(runner.state.prep_failure.is_some());
15622        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15623        runner.state = RunState::load(&runner.state.id).unwrap();
15624        assert!(
15625            runner.prep().await.is_err(),
15626            "correcting the cause alone must not auto-retry"
15627        );
15628        let failed_at = runner.state.events.last().unwrap().at;
15629        assert!(
15630            runner.retry_preparation(failed_at).is_err(),
15631            "old approvals cannot unlock the hold"
15632        );
15633        let mut task = crate::queue::Task::new(
15634            "retry".into(),
15635            "retry setup".into(),
15636            repo.clone(),
15637            crate::queue::Source::Human,
15638        );
15639        task.start(runner.state.id.clone());
15640        task.hold_machine(Some("setup failed".into()));
15641        task.release_operator("cli", None);
15642        runner
15643            .retry_preparation(task.resumption.as_ref().unwrap().at)
15644            .unwrap();
15645        runner.prep().await.unwrap();
15646        assert!(runner.state.prep_failure.is_none());
15647        assert_eq!(runner.state.prep_complete, Some(true));
15648        assert_eq!(runner.state.candidates.len(), 2);
15649        assert!(runner.state.worktree_root().join("judge-1").exists());
15650        for candidate in &runner.state.candidates {
15651            assert_eq!(
15652                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15653                git::tree_of(&repo, &seed).await.unwrap()
15654            );
15655        }
15656    }
15657
15658    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15659        let mut heads = Vec::new();
15660        for candidate in candidates {
15661            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15662        }
15663        heads
15664    }
15665
15666    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15667        ask_test_home();
15668        let tmp = tempfile::tempdir().unwrap();
15669        let repo = tmp.path().join("repo");
15670        std::fs::create_dir(&repo).unwrap();
15671        init_repo(&repo);
15672        let mut runner = runner_at(RunStatus::Reviewing);
15673        runner.state.repo = repo.clone();
15674        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15675        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15676        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15677            let branch = format!("candidate-{label}");
15678            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15679            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15680                .await
15681                .unwrap();
15682            if label == 'B' {
15683                std::fs::write(worktree.join("change"), "winner").unwrap();
15684                git::rescue_commit(&worktree, "candidate change")
15685                    .await
15686                    .unwrap();
15687            }
15688            runner.state.candidates.push(Candidate {
15689                index,
15690                label,
15691                agent: "alpha".into(),
15692                branch,
15693                worktree,
15694                summary: "completed reply".into(),
15695                stat: String::new(),
15696                files: usize::from(label == 'B'),
15697                commits: usize::from(label == 'B'),
15698                empty: label == 'A',
15699                failed: None,
15700                verified_noop: None,
15701                duration_ms: 1,
15702                folded: false,
15703            });
15704        }
15705        runner.state.judge_skipped = true;
15706        runner.state.tally = Some(Tally {
15707            first_choice: BTreeMap::new(),
15708            borda: BTreeMap::new(),
15709            winner: 'B',
15710            rankings: 0,
15711            unanimous_initial: false,
15712            deliberated: false,
15713            changed_votes: 0,
15714            unanimous_final: false,
15715            tie_break: None,
15716            judges: 0,
15717            present: 0,
15718            quorum: 0,
15719            met_quorum: true,
15720            uncontested: Some("only B produced a change".into()),
15721        });
15722        runner.state.save().unwrap();
15723        (tmp, runner)
15724    }
15725
15726    #[tokio::test]
15727    #[cfg(unix)]
15728    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15729        for (solo, failed_checkpoint) in
15730            [(true, false), (false, false), (true, true), (false, true)]
15731        {
15732            let (_tmp, mut runner) = preservation_resume_fixture().await;
15733            if solo {
15734                runner.state.candidates.retain(|c| c.label == 'A');
15735            }
15736            runner.state.candidates[0].empty = false;
15737            runner.state.candidates[0].summary.clear();
15738            runner.state.tally = None;
15739            runner.state.judge_skipped = false;
15740            runner.state.status = RunStatus::Implementing;
15741            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15742            let mut implementer = spec("alpha");
15743            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15744            runner.roles.implementers = vec![implementer];
15745            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15746            if failed_checkpoint {
15747                std::fs::create_dir_all(&patch).unwrap();
15748            }
15749            runner.state.save().unwrap();
15750            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15751            drop(pressure);
15752            runner.state = RunState::load(&runner.state.id).unwrap();
15753            assert_eq!(runner.state.status, RunStatus::Blocked);
15754            assert!(runner.state.interrupted_implementations.contains(&'A'));
15755            assert_eq!(runner.state.candidates[0].commits, 0);
15756            assert!(!runner.state.candidates[0].empty);
15757            assert!(runner.state.candidates[0].failed.is_none());
15758            if failed_checkpoint {
15759                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15760                std::fs::remove_dir(patch).unwrap();
15761            }
15762            runner.state.resource_block = None;
15763            runner.state.config.disk.min_free_bytes = 0;
15764            runner.roles.implementers[0].command = vec![
15765                "sh".into(),
15766                "-c".into(),
15767                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15768            ];
15769            runner.implement().await.unwrap();
15770            assert!(runner.state.interrupted_implementations.is_empty());
15771            assert!(runner.state.candidate_preservation.is_empty());
15772            assert_eq!(runner.state.candidates[0].commits, 1);
15773            assert!(!runner.state.candidates[0].empty);
15774            assert!(
15775                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15776                    .unwrap()
15777                    .contains("retried empty interrupted seat")
15778            );
15779            if !solo {
15780                assert_eq!(runner.state.candidates[1].commits, 1);
15781            }
15782        }
15783    }
15784
15785    #[tokio::test]
15786    #[cfg(unix)]
15787    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15788        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15789            let (_tmp, mut runner) = preservation_resume_fixture().await;
15790            runner.state.candidates.retain(|c| c.label == 'A');
15791            runner.state.candidates[0].empty = false;
15792            runner.state.candidates[0].summary.clear();
15793            runner.state.tally = None;
15794            runner.state.judge_skipped = false;
15795            runner.state.status = RunStatus::Implementing;
15796            // Force only the real post-spawn watchdog, without failing the
15797            // separate admission check. Existing edits predate its measurement.
15798            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15799            let wt = runner.state.candidates[0].worktree.clone();
15800            std::fs::write(
15801                wt.join("interrupted-source"),
15802                "retain implementation work\n",
15803            )
15804            .unwrap();
15805            if committed {
15806                git::rescue_commit(&wt, "implementation checkpoint")
15807                    .await
15808                    .unwrap();
15809            }
15810            let lock = git::git(
15811                &wt,
15812                &[
15813                    "rev-parse",
15814                    "--path-format=absolute",
15815                    "--git-path",
15816                    "index.lock",
15817                ],
15818            )
15819            .await
15820            .unwrap();
15821            if block_rescue {
15822                std::fs::write(&lock, "fixture-owned lock").unwrap();
15823            }
15824            let mut implementer = spec("alpha");
15825            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15826            runner.roles.implementers = vec![implementer];
15827            runner.state.save().unwrap();
15828            let result = runner.implement().await;
15829            drop(pressure);
15830            assert_eq!(result.is_err(), block_rescue);
15831            assert_eq!(
15832                runner.state.resource_block.as_ref().unwrap().kind,
15833                "disk pressure"
15834            );
15835            assert!(
15836                runner
15837                    .state
15838                    .events
15839                    .iter()
15840                    .any(|e| e.message.contains("agent execution interrupted"))
15841            );
15842            assert!(
15843                runner
15844                    .state
15845                    .dir()
15846                    .join("artifacts/candidate-A-recovery/staged.patch")
15847                    .exists()
15848            );
15849            if block_rescue {
15850                let saved = RunState::load(&runner.state.id).unwrap();
15851                assert!(saved.candidate_preservation.contains_key(&'A'));
15852                assert!(!saved.candidates[0].empty);
15853                assert_eq!(
15854                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15855                    "retain implementation work\n"
15856                );
15857                std::fs::remove_file(lock).unwrap();
15858                runner.state = saved;
15859            } else {
15860                assert_eq!(runner.state.status, RunStatus::Blocked);
15861                assert_eq!(runner.state.candidates[0].commits, 1);
15862            }
15863            // Recovery must save the candidate before deciding whether another
15864            // implementation is needed. No implementer is available on retry.
15865            runner.roles.implementers.clear();
15866            runner.state.config.disk.min_free_bytes = 0;
15867            runner.state.resource_block = None;
15868            runner.implement().await.unwrap();
15869            assert!(runner.state.candidate_preservation.is_empty());
15870            assert_eq!(runner.state.candidates[0].commits, 1);
15871            assert!(!runner.state.candidates[0].empty);
15872            assert!(git::is_clean(&wt).await.unwrap());
15873            assert!(
15874                crate::run::read_artifact(&runner.state, "cand-A.patch")
15875                    .unwrap()
15876                    .contains("retain implementation work")
15877            );
15878        }
15879    }
15880
15881    #[tokio::test]
15882    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15883        let (_tmp, mut runner) = preservation_resume_fixture().await;
15884        // Schema 15 already confirmed the unfolded no-change result.
15885        runner.implement().await.unwrap();
15886        assert!(runner.state.candidate_preservation.is_empty());
15887        assert!(
15888            !runner
15889                .state
15890                .dir()
15891                .join("artifacts/candidate-A-recovery")
15892                .exists()
15893        );
15894        runner.fold_losers().await.unwrap();
15895        assert!(!runner.state.candidates[0].worktree.exists());
15896        for schema in [14, crate::run::SCHEMA] {
15897            runner.state.schema = schema;
15898            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15899            runner.state.save().unwrap();
15900            runner.state = RunState::load(&runner.state.id).unwrap();
15901            runner.implement().await.unwrap();
15902            assert!(runner.state.candidate_preservation.is_empty());
15903            assert!(runner.state.candidates[0].folded);
15904            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15905        }
15906    }
15907
15908    #[tokio::test]
15909    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15910        let (_tmp, mut runner) = preservation_resume_fixture().await;
15911        runner.state.candidates.retain(|c| c.label == 'A');
15912        runner.state.tally = None;
15913        runner.state.judge_skipped = false;
15914        let candidate = &mut runner.state.candidates[0];
15915        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15916        candidate.failed = Some("deployment validation unavailable".into());
15917        std::fs::write(
15918            candidate.worktree.join("repair"),
15919            "future launcher policy\n",
15920        )
15921        .unwrap();
15922        runner
15923            .state
15924            .candidate_preservation
15925            .insert('A', "awaiting persistence".into());
15926        runner.persist_candidates().await.unwrap();
15927        let candidate = &runner.state.candidates[0];
15928        assert!(!candidate.empty);
15929        assert_eq!(candidate.commits, 1);
15930        assert_eq!(candidate.files, 1);
15931        assert!(candidate.failed.is_none());
15932        assert!(candidate.summary.contains("EPERM"));
15933        assert!(
15934            crate::run::read_artifact(&runner.state, "cand-A.patch")
15935                .unwrap()
15936                .contains("future launcher policy")
15937        );
15938        assert!(runner.state.candidate_preservation.is_empty());
15939    }
15940
15941    #[tokio::test]
15942    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
15943        let (_tmp, mut runner) = preservation_resume_fixture().await;
15944        let wt = runner.state.candidates[0].worktree.clone();
15945        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
15946        runner.state.schema = 14;
15947        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15948        assert!(runner.state.candidate_preservation.contains_key(&'A'));
15949        let tally = serde_json::to_value(&runner.state.tally).unwrap();
15950        for _ in 0..2 {
15951            assert!(runner.implement().await.is_err());
15952            assert_eq!(runner.state.status, RunStatus::Blocked);
15953            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
15954            assert!(
15955                runner.state.candidates[0].empty,
15956                "old verdict must remain unusable"
15957            );
15958            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
15959            assert!(runner.state.judge_skipped);
15960            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
15961            assert!(
15962                crate::run::read_artifact(&runner.state, "cand-A.patch")
15963                    .unwrap()
15964                    .contains("owned edits")
15965            );
15966            assert!(wt.exists());
15967            assert_eq!(
15968                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
15969                    .await
15970                    .unwrap(),
15971                1
15972            );
15973            runner.state = RunState::load(&runner.state.id).unwrap();
15974        }
15975    }
15976
15977    #[tokio::test]
15978    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
15979        let (_tmp, mut runner) = preservation_resume_fixture().await;
15980        let wt = runner.state.candidates[0].worktree.clone();
15981        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
15982        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
15983        let index_lock = PathBuf::from(
15984            git::git(
15985                &wt,
15986                &[
15987                    "rev-parse",
15988                    "--path-format=absolute",
15989                    "--git-path",
15990                    "index.lock",
15991                ],
15992            )
15993            .await
15994            .unwrap(),
15995        );
15996        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
15997        runner.state.schema = 14;
15998        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15999        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16000        assert!(runner.implement().await.is_err());
16001        assert!(runner.after_implement().is_err());
16002        assert!(runner.fold_losers().await.is_err());
16003        assert_eq!(
16004            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16005            "failed seat edits"
16006        );
16007        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16008        runner.state = RunState::load(&runner.state.id).unwrap();
16009        std::fs::remove_file(index_lock).unwrap();
16010        // Recovery finds changes that the old uncontested result never judged.
16011        assert!(runner.implement().await.is_err());
16012        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16013        assert!(runner.state.candidates[0].empty);
16014        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16015        assert!(
16016            crate::run::read_artifact(&runner.state, "cand-A.patch")
16017                .unwrap()
16018                .contains("failed seat edits")
16019        );
16020        assert!(wt.exists());
16021    }
16022
16023    #[tokio::test]
16024    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16025        let (_tmp, mut runner) = preservation_resume_fixture().await;
16026        let repo = runner.state.repo.clone();
16027        let common = git::git(
16028            &repo,
16029            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16030        )
16031        .await
16032        .unwrap();
16033        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16034        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16035        assert!(runner.fold_losers().await.is_err());
16036        assert!(!runner.state.candidates[0].worktree.exists());
16037        assert!(!runner.state.candidates[0].folded);
16038        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16039        runner.state = RunState::load(&runner.state.id).unwrap();
16040        std::fs::remove_file(branch_lock).unwrap();
16041        let judge = runner.state.worktree_root().join("judge-1");
16042        git::worktree_add_detached(&repo, &judge, "main")
16043            .await
16044            .unwrap();
16045        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16046        runner.roles.judges = vec![spec("judge")];
16047        assert!(runner.fold_losers().await.is_err());
16048        runner.state = RunState::load(&runner.state.id).unwrap();
16049        assert!(
16050            runner.state.candidates[0].folded,
16051            "candidate progress saved before judge failure"
16052        );
16053        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16054        assert!(judge.join("uncommitted").exists());
16055        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16056        runner.fold_losers().await.unwrap();
16057        assert!(!judge.exists());
16058        runner.fold_losers().await.unwrap();
16059    }
16060
16061    #[tokio::test]
16062    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16063        let (_tmp, mut runner) = preservation_resume_fixture().await;
16064        runner.state.candidates.retain(|c| c.label == 'B');
16065        runner.state.tally = None;
16066        runner.state.judge_skipped = false;
16067        runner.state.prep_complete = Some(true);
16068        runner.state.config.graph.advise = false;
16069        runner.state.config.graph.selected_implementer = Some("alpha".into());
16070        let wt = runner.state.candidates[0].worktree.clone();
16071        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16072        runner
16073            .state
16074            .candidate_preservation
16075            .insert('B', "awaiting persistence".into());
16076        let index_lock = git::git(
16077            &wt,
16078            &[
16079                "rev-parse",
16080                "--path-format=absolute",
16081                "--git-path",
16082                "index.lock",
16083            ],
16084        )
16085        .await
16086        .unwrap();
16087        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16088        runner.state.save().unwrap();
16089        assert!(runner.execute_graph().await.is_err());
16090        assert_eq!(runner.state.status, RunStatus::Blocked);
16091        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16092        std::fs::remove_file(index_lock).unwrap();
16093        // Park at the next graph boundary: this tests the real selected-seat
16094        // hold guard without dispatching reviewers or publishing anything.
16095        for _ in 0..2 {
16096            runner.state = RunState::load(&runner.state.id).unwrap();
16097            runner.pause.park();
16098            runner.execute_graph().await.unwrap();
16099            assert_eq!(runner.state.status, RunStatus::Implementing);
16100            assert!(runner.state.candidate_preservation.is_empty());
16101            assert_eq!(runner.state.candidates[0].commits, 2);
16102            assert!(runner.state.tally.is_none());
16103        }
16104        runner.implement().await.unwrap();
16105        assert_eq!(runner.state.status, RunStatus::Judging);
16106        // A genuine selected-implementer hold must still require release.
16107        runner.state.status = RunStatus::Blocked;
16108        runner.state.candidates[0].failed = Some("implementation failed".into());
16109        runner.state.save().unwrap();
16110        let error = runner.execute_graph().await.unwrap_err();
16111        assert!(
16112            error
16113                .to_string()
16114                .contains("selected implementation is held")
16115        );
16116    }
16117
16118    #[tokio::test]
16119    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16120        let (_tmp, mut runner) = preservation_resume_fixture().await;
16121        let repo = runner.state.repo.clone();
16122        let candidate = runner.state.candidates[0].clone();
16123        // Remove only the disposable fixture directory, leaving Git's linked
16124        // worktree registration and branch intact, as in a vanished checkout.
16125        let registered = git::worktree_holding(&repo, &candidate.branch)
16126            .await
16127            .unwrap()
16128            .unwrap();
16129        assert_eq!(
16130            registered.canonicalize().unwrap(),
16131            candidate.worktree.canonicalize().unwrap()
16132        );
16133        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16134        assert_eq!(
16135            git::worktree_holding(&repo, &candidate.branch)
16136                .await
16137                .unwrap(),
16138            Some(registered)
16139        );
16140        assert!(
16141            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16142            "the stale registration initially keeps the branch checked out"
16143        );
16144        runner.fold_losers().await.unwrap();
16145        assert!(runner.state.candidates[0].folded);
16146        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16147        assert_eq!(
16148            git::worktree_holding(&repo, &candidate.branch)
16149                .await
16150                .unwrap(),
16151            None
16152        );
16153        runner.state = RunState::load(&runner.state.id).unwrap();
16154        runner.fold_losers().await.unwrap();
16155    }
16156
16157    #[tokio::test]
16158    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16159        let (_tmp, mut runner) = preservation_resume_fixture().await;
16160        let repo = runner.state.repo.clone();
16161        let candidate = runner.state.candidates[0].clone();
16162        let winner = runner.state.candidates[1].clone();
16163        let registered = git::worktree_holding(&repo, &candidate.branch)
16164            .await
16165            .unwrap()
16166            .unwrap();
16167        assert_eq!(
16168            registered.canonicalize().unwrap(),
16169            candidate.worktree.canonicalize().unwrap()
16170        );
16171        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16172        assert_eq!(
16173            git::worktree_holding(&repo, &candidate.branch)
16174                .await
16175                .unwrap(),
16176            Some(registered)
16177        );
16178        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16179
16180        for _ in 0..2 {
16181            fold_run(&mut runner.state, false, &crate::run::home())
16182                .await
16183                .unwrap();
16184            runner.state = RunState::load(&runner.state.id).unwrap();
16185            assert!(runner.state.candidates[0].folded);
16186            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16187            assert_eq!(
16188                git::worktree_holding(&repo, &candidate.branch)
16189                    .await
16190                    .unwrap(),
16191                None
16192            );
16193            assert!(!runner.state.candidates[1].folded);
16194            assert!(winner.worktree.exists());
16195            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16196        }
16197    }
16198
16199    #[tokio::test]
16200    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16201        ask_test_home();
16202        let tmp = tempfile::tempdir().unwrap();
16203        let repo = tmp.path().join("repo");
16204        std::fs::create_dir(&repo).unwrap();
16205        init_repo(&repo);
16206        let mut runner = runner_at(RunStatus::Implementing);
16207        runner.state.repo = repo.clone();
16208        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16209        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16210        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16211            let branch = format!("candidate-{label}");
16212            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16213            git::worktree_add_branch(&repo, &wt, &branch, "main")
16214                .await
16215                .unwrap();
16216            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16217            git::git(&wt, &["add", "README.md"]).await.unwrap();
16218            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16219            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16220            runner.state.candidates.push(Candidate {
16221                index,
16222                label,
16223                agent: "alpha".to_owned(),
16224                branch,
16225                worktree: wt,
16226                summary: "completed implementation".to_owned(),
16227                stat: String::new(),
16228                files: 0,
16229                commits: 0,
16230                empty: false,
16231                failed: None,
16232                verified_noop: None,
16233                duration_ms: 42,
16234                folded: false,
16235            });
16236            runner
16237                .state
16238                .candidate_preservation
16239                .insert(label, "awaiting persistence".to_owned());
16240        }
16241        let wt = runner.state.candidates[0].worktree.clone();
16242        let index_lock = git::git(
16243            &wt,
16244            &[
16245                "rev-parse",
16246                "--path-format=absolute",
16247                "--git-path",
16248                "index.lock",
16249            ],
16250        )
16251        .await
16252        .unwrap();
16253        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16254        runner.state.save().unwrap();
16255        assert!(runner.persist_candidates().await.is_err());
16256        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16257        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16258        assert_eq!(
16259            runner.state.candidates[1].commits, 1,
16260            "other results are still collected"
16261        );
16262        assert!(
16263            !runner.state.candidates[0].empty,
16264            "failed rescue is not no change"
16265        );
16266        assert_eq!(
16267            runner.state.candidates[0].summary,
16268            "completed implementation"
16269        );
16270        assert!(runner.after_implement().is_err());
16271        let home = tmp.path().join("fold-home");
16272        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16273        assert!(wt.join("new-file").exists());
16274        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16275        assert!(recovery.join("staged.patch").exists());
16276        assert!(recovery.join("unstaged.patch").exists());
16277        assert_eq!(
16278            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16279            "untracked edits\n"
16280        );
16281        runner.state.begin_pass();
16282        runner.state.end_pass_parked(ParkCause::Upgrade {
16283            from: "1.0".into(),
16284            to: Some("1.1".into()),
16285        });
16286        runner.state.save().unwrap();
16287        runner.state = RunState::load(&runner.state.id).unwrap();
16288        assert!(matches!(
16289            runner.state.passes.last().unwrap().end,
16290            Some(crate::run::PassEnd::Parked {
16291                cause: ParkCause::Upgrade { .. }
16292            })
16293        ));
16294        std::fs::remove_file(index_lock).unwrap();
16295        // No implementer is configured: a persistence retry cannot dispatch a seat.
16296        runner.implement().await.unwrap();
16297        assert!(runner.state.candidate_preservation.is_empty());
16298        assert!(
16299            runner
16300                .state
16301                .candidates
16302                .iter()
16303                .all(|c| c.commits == 1 && !c.empty)
16304        );
16305        assert!(git::is_clean(&wt).await.unwrap());
16306        assert!(
16307            crate::run::read_artifact(&runner.state, "cand-A.patch")
16308                .unwrap()
16309                .contains("later tracked edits")
16310        );
16311    }
16312
16313    #[tokio::test]
16314    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16315        ask_test_home();
16316        let tmp = tempfile::tempdir().unwrap();
16317        let repo = tmp.path().join("repo");
16318        std::fs::create_dir(&repo).unwrap();
16319        init_repo(&repo);
16320        let wt = tmp.path().join("candidate");
16321        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16322            .await
16323            .unwrap();
16324        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16325        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16326        assert!(wt.join("uncommitted").exists());
16327        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16328        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16329        let mut runner = runner_at(RunStatus::Implementing);
16330        runner.state.repo = repo.clone();
16331        runner.state.base_commit = "missing-base".to_owned();
16332        runner.state.candidates.push(Candidate {
16333            index: 0,
16334            label: 'A',
16335            agent: "alpha".into(),
16336            branch: "candidate".into(),
16337            worktree: wt.clone(),
16338            summary: "completed".into(),
16339            stat: String::new(),
16340            files: 0,
16341            commits: 0,
16342            empty: false,
16343            failed: None,
16344            verified_noop: None,
16345            duration_ms: 1,
16346            folded: false,
16347        });
16348        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16349        assert!(
16350            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16351                .await
16352                .is_err()
16353        );
16354        assert!(!runner.state.candidates[0].folded);
16355        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16356        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16357        runner
16358            .state
16359            .candidate_preservation
16360            .insert('A', "pending".into());
16361        assert!(runner.persist_candidates().await.is_err());
16362        assert!(!runner.state.candidates[0].empty);
16363        assert!(!runner.state.candidates[0].folded);
16364        assert!(wt.exists());
16365        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16366        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16367        std::fs::create_dir_all(&patch_path).unwrap();
16368        assert!(
16369            runner.persist_candidates().await.is_err(),
16370            "artifact failure stays pending"
16371        );
16372        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16373        assert!(!runner.state.candidates[0].empty);
16374        std::fs::remove_dir(patch_path).unwrap();
16375        runner.persist_candidates().await.unwrap();
16376        assert!(
16377            runner.state.candidates[0].empty,
16378            "only successful inspection confirms no change"
16379        );
16380    }
16381
16382    fn init_repo(dir: &Path) {
16383        let run = |args: &[&str]| {
16384            let out = std::process::Command::new("git")
16385                .args(args)
16386                .current_dir(dir)
16387                .quiet()
16388                .output()
16389                .expect("spawn git");
16390            assert!(
16391                out.status.success(),
16392                "git {args:?} failed: {}",
16393                String::from_utf8_lossy(&out.stderr)
16394            );
16395        };
16396        run(&["init", "-b", "main"]);
16397        run(&["config", "user.name", "magi test"]);
16398        run(&["config", "user.email", "magi@example.com"]);
16399        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16400        run(&["add", "-A"]);
16401        run(&["commit", "-m", "init"]);
16402    }
16403
16404    // `settle_questions` is what closes the ghost the phone showed: a run's
16405    // seat asked something, the run then ended, and nothing was left to
16406    // abandon the question it left `open`. `HOME` is a process-wide
16407    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16408    // first time it runs in the binary — every test below still reaches the
16409    // same directory whichever call won, and each gets its own run id from
16410    // `RunState::new`, so they never collide there.
16411    fn ask_test_home() {
16412        crate::run::pin_test_home();
16413    }
16414
16415    #[test]
16416    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16417        ask_test_home();
16418        let mut runner = runner_at(RunStatus::Prep);
16419        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16420        // A blocked temporary output path makes the terminal save fail while
16421        // preserving the already published reservation.
16422        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16423        let error = guard
16424            .finish(
16425                &mut runner.state,
16426                Err(
16427                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16428                        .into(),
16429                ),
16430            )
16431            .unwrap_err();
16432        assert_eq!(
16433            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16434            std::io::ErrorKind::PermissionDenied
16435        );
16436        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16437        let message = format!("{error:#}");
16438        assert!(message.contains("original refusal"));
16439        assert!(!message.contains("could not save terminal"));
16440        assert_eq!(
16441            error.chain().count(),
16442            2,
16443            "only startup attribution wraps the original error"
16444        );
16445        let saved = RunState::load(&runner.state.id).unwrap();
16446        assert_eq!(saved.status, RunStatus::Blocked);
16447        assert!(!saved.startup.unwrap().complete);
16448    }
16449
16450    #[test]
16451    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16452        ask_test_home();
16453        let mut runner = runner_at(RunStatus::Prep);
16454        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16455        let id = runner.state.id.clone();
16456        drop(guard);
16457        let saved = RunState::load(&id).unwrap();
16458        assert_eq!(saved.status, RunStatus::Blocked);
16459        assert!(!saved.startup.as_ref().unwrap().complete);
16460        assert!(
16461            saved
16462                .startup
16463                .unwrap()
16464                .failure
16465                .unwrap()
16466                .contains("interrupted")
16467        );
16468        assert!(Runner::resume(&id).is_err());
16469    }
16470
16471    #[test]
16472    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16473        ask_test_home();
16474        let mut runner = runner_at(RunStatus::Prep);
16475        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16476        let error = guard
16477            .finish(
16478                &mut runner.state,
16479                Err(
16480                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16481                        .into(),
16482                ),
16483            )
16484            .unwrap_err();
16485        assert_eq!(
16486            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16487            std::io::ErrorKind::PermissionDenied
16488        );
16489        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16490        let mut task = crate::queue::Task::new(
16491            "review".into(),
16492            "criteria".into(),
16493            runner.state.repo.clone(),
16494            crate::queue::Source::Human,
16495        );
16496        task.record_startup_failure(failure.clone());
16497        task.record_startup_failure(failure.clone());
16498        assert_eq!(task.startup_failures.len(), 1);
16499        assert!(task.runs.is_empty());
16500        assert_eq!(task.attempts, 0);
16501    }
16502
16503    #[cfg(unix)]
16504    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16505        ask_test_home();
16506        let tmp = tempfile::tempdir().unwrap();
16507        let repo = tmp.path().join("repo");
16508        std::fs::create_dir(&repo).unwrap();
16509        init_repo(&repo);
16510        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16511        let origin = tmp.path().join("origin.git");
16512        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16513            .await
16514            .unwrap();
16515        git::git(
16516            &repo,
16517            &["remote", "add", "origin", origin.to_str().unwrap()],
16518        )
16519        .await
16520        .unwrap();
16521        git::git(&repo, &["checkout", "-b", "fix/retained"])
16522            .await
16523            .unwrap();
16524        std::fs::write(repo.join("retained"), "retained change").unwrap();
16525        git::git(&repo, &["add", "."]).await.unwrap();
16526        git::git(&repo, &["commit", "-m", "retained implementation"])
16527            .await
16528            .unwrap();
16529        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16530        git::git(&repo, &["checkout", "main"]).await.unwrap();
16531        std::fs::write(repo.join("base-change"), "new base").unwrap();
16532        git::git(&repo, &["add", "."]).await.unwrap();
16533        git::git(&repo, &["commit", "-m", "base moved"])
16534            .await
16535            .unwrap();
16536        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16537        git::git(&repo, &["checkout", "fix/retained"])
16538            .await
16539            .unwrap();
16540        let mut runner = runner_at(RunStatus::Voting);
16541        runner.state.repo = repo.clone();
16542        runner.state.base_commit = base;
16543        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16544        runner.state.config.disk.min_free_bytes = 0;
16545        runner.state.config.graph.review_rounds = 1;
16546        runner.state.config.verify.e2e = vec!["true".into()];
16547        runner.state.config.verify.gate = vec!["true".into()];
16548        runner.state.candidates = vec![Candidate {
16549            index: 0,
16550            label: 'A',
16551            agent: "fixture".into(),
16552            branch: "fix/retained".into(),
16553            worktree: repo.clone(),
16554            summary: String::new(),
16555            stat: String::new(),
16556            files: 1,
16557            commits: 1,
16558            empty: false,
16559            failed: None,
16560            verified_noop: None,
16561            duration_ms: 0,
16562            folded: false,
16563        }];
16564        runner.state.tally = Some(Tally {
16565            first_choice: BTreeMap::new(),
16566            borda: BTreeMap::new(),
16567            winner: 'A',
16568            rankings: 0,
16569            unanimous_initial: true,
16570            deliberated: false,
16571            changed_votes: 0,
16572            unanimous_final: true,
16573            tie_break: None,
16574            judges: 0,
16575            present: 0,
16576            quorum: 0,
16577            met_quorum: true,
16578            uncontested: Some("review-only".into()),
16579        });
16580        let mut task = crate::queue::Task::new(
16581            "fixture".into(),
16582            "fixture".into(),
16583            repo,
16584            crate::queue::Source::Human,
16585        );
16586        task.link_run(&runner.state.id);
16587        crate::queue::Queue::open().put(&mut task).unwrap();
16588        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16589        runner.state.base_sync = Some(BaseSync {
16590            tip: String::new(),
16591            behind: 1,
16592            attempts: BASE_SYNC_ROUNDS,
16593            conflict: None,
16594            already_in: None,
16595        });
16596        runner.sync_to_base().await.unwrap();
16597        assert_eq!(runner.state.status, RunStatus::Blocked);
16598        (tmp, runner, head, task.id)
16599    }
16600
16601    #[cfg(unix)]
16602    #[tokio::test]
16603    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16604        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16605        if std::env::var_os(CHILD).is_none() {
16606            let home = tempfile::tempdir().unwrap();
16607            let output = std::process::Command::new(std::env::current_exe().unwrap())
16608                .args([
16609                    "--exact",
16610                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16611                    "--nocapture",
16612                ])
16613                .env(CHILD, "1")
16614                .env("MAGI_HOME", home.path())
16615                .env("MAGI_WORKER_HOME", home.path())
16616                .env("MAGI_WORKER_ID", "abcdef")
16617                .output()
16618                .unwrap();
16619            assert!(
16620                output.status.success()
16621                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16622                "{}\n{}",
16623                String::from_utf8_lossy(&output.stdout),
16624                String::from_utf8_lossy(&output.stderr)
16625            );
16626            return;
16627        }
16628        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16629        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16630        let home = crate::run::home();
16631        let queue = crate::queue::Queue::open();
16632        let mut owner = queue.get(&task).unwrap();
16633        owner.release_operator("test-release", None);
16634        owner.start(runner.state.id.clone());
16635        queue.put(&mut owner).unwrap();
16636        runner.state.driver_exited = true;
16637        runner.state.task_instruction = Some(owner.instruction.clone());
16638        runner.state.save().unwrap();
16639        let _claim = queue.claim(&task).unwrap();
16640        let reserve = |id: &str| {
16641            let directory = home.join("workers/jobs").join(id);
16642            std::fs::create_dir_all(&directory).unwrap();
16643            let job = crate::worker::Job {
16644                id: id.into(),
16645                sequence: 1,
16646                kind: "task".into(),
16647                key: format!("task:{task}"),
16648                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16649                generation: "fixture-generation".into(),
16650                version: None,
16651                status: crate::worker::Status::Running,
16652                result: None,
16653                pid: Some(std::process::id()),
16654                created_at: 0,
16655                finished_at: None,
16656            };
16657            std::fs::write(
16658                directory.join("job.json"),
16659                serde_json::to_vec(&job).unwrap(),
16660            )
16661            .unwrap();
16662            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16663        };
16664        let _own = reserve("abcdef");
16665        let other = reserve("fedcba");
16666        assert!(
16667            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16668                .await
16669                .is_err()
16670        );
16671        assert!(
16672            RunState::load(&runner.state.id)
16673                .unwrap()
16674                .base_sync_recovery
16675                .authorizations
16676                .is_empty()
16677        );
16678        drop(other);
16679        assert!(crate::daemon::is_working_on(
16680            &home,
16681            &runner.state.id,
16682            Timestamp::now()
16683        ));
16684        assert!(
16685            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16686        );
16687        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16688            .await
16689            .unwrap();
16690        assert!(
16691            RunState::load(&runner.state.id)
16692                .unwrap()
16693                .base_recovery_pending()
16694        );
16695    }
16696
16697    #[cfg(unix)]
16698    #[tokio::test]
16699    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16700        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16701        let home = crate::run::home();
16702        runner.state.driver_pid = Some(std::process::id());
16703        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16704        runner.state.driver_exited = false;
16705        runner.state.begin_pass();
16706        runner.state.save().unwrap();
16707        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16708        driver.record(&runner.state).unwrap();
16709        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16710        // Preserve the durable claim as if the process died before Drop saved.
16711        drop(driver);
16712        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16713        assert_eq!(
16714            runner.state.liveness_under(&home, false),
16715            crate::run::Liveness::Dead
16716        );
16717        let queue = crate::queue::Queue::open();
16718        let _claim = queue.claim(&task).unwrap();
16719        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16720            .await
16721            .unwrap();
16722        let disk = RunState::load(&runner.state.id).unwrap();
16723        assert!(!disk.driver_exited);
16724        assert!(disk.base_recovery_pending());
16725    }
16726
16727    #[cfg(unix)]
16728    #[tokio::test]
16729    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16730        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16731        if std::env::var_os(CHILD).is_none() {
16732            use std::os::unix::fs::PermissionsExt;
16733            let tmp = tempfile::tempdir().unwrap();
16734            let gh = tmp.path().join("gh");
16735            std::fs::write(&gh, r#"#!/bin/sh
16736case "$1 $2" in
16737  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16738  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16739  *) exit 2 ;;
16740esac
16741"#).unwrap();
16742            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16743            let mut paths = vec![tmp.path().to_path_buf()];
16744            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16745            let output = std::process::Command::new(std::env::current_exe().unwrap())
16746                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16747                .env(CHILD, "1")
16748                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16749                .env("PATH", std::env::join_paths(paths).unwrap())
16750                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16751                .output().unwrap();
16752            assert!(
16753                output.status.success()
16754                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16755                "{}\n{}",
16756                String::from_utf8_lossy(&output.stdout),
16757                String::from_utf8_lossy(&output.stderr)
16758            );
16759            return;
16760        }
16761        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16762        let repo = runner.state.repo.clone();
16763        git::git(&repo, &["checkout", "main"]).await.unwrap();
16764        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16765        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16766        git::git(&repo, &["checkout", "fix/retained"])
16767            .await
16768            .unwrap();
16769        runner.state.pr = Some(crate::run::PrRecord {
16770            url: "https://example.test/project/pull/1".into(),
16771            number: 1,
16772            state: "open".into(),
16773            checks: "unknown".into(),
16774            round: 0,
16775            rounds: 1,
16776            red_at_merge: Vec::new(),
16777        });
16778        runner.sync_to_base().await.unwrap();
16779        assert_eq!(runner.state.status, RunStatus::Blocked);
16780        assert_eq!(
16781            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16782            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16783        );
16784        assert!(runner.state.base_sync_closure_pending());
16785        assert!(crate::handover::can_resume(&runner.state));
16786        let queue = crate::queue::Queue::open();
16787        assert!(!crate::base_sync_recovery::held_task(
16788            &queue,
16789            &queue.get(&task).unwrap()
16790        ));
16791        assert!(runner.state.authorize_base_sync(&task).is_err());
16792        // Legacy closure stops have no typed stop but must retain the checkpoint.
16793        runner.state.base_sync_recovery.stop = None;
16794        runner.state.status = RunStatus::Voting;
16795        runner.state.gate_ran = true;
16796        runner.state.config.graph.review_rounds = 0;
16797        runner.state.save().unwrap();
16798        assert!(crate::handover::can_resume(&runner.state));
16799        runner.state = RunState::load(&runner.state.id).unwrap();
16800        runner.execute_graph().await.unwrap();
16801        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16802        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16803        assert_eq!(
16804            runner.state.base_sync.as_ref().unwrap().attempts,
16805            BASE_SYNC_ROUNDS
16806        );
16807        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16808        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16809        assert!(runner.state.reviews.is_empty());
16810        assert!(runner.state.gate.is_empty());
16811    }
16812
16813    #[cfg(unix)]
16814    #[tokio::test]
16815    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16816        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16817        runner.state.driver_exited = true;
16818        runner.state.save().unwrap();
16819        let home = crate::run::home();
16820        let queue = crate::queue::Queue::open();
16821        let _claim = queue.claim(&task).unwrap();
16822        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16823        assert!(
16824            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16825                .await
16826                .is_err()
16827        );
16828        drop(lease);
16829        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16830        assert!(
16831            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16832                .await
16833                .is_err()
16834        );
16835        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16836        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16837            .await
16838            .unwrap();
16839        let disk = RunState::load(&runner.state.id).unwrap();
16840        assert!(disk.base_recovery_pending());
16841        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16842        assert!(
16843            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16844                .await
16845                .is_err()
16846        );
16847        assert_eq!(
16848            RunState::load(&runner.state.id)
16849                .unwrap()
16850                .base_sync_recovery
16851                .authorizations
16852                .len(),
16853            1
16854        );
16855        let mut owner = queue.get(&task).unwrap();
16856        owner.runs.push("20260101-000000-ffff".into());
16857        queue.put(&mut owner).unwrap();
16858        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16859            .await
16860            .unwrap_err();
16861        assert!(error.to_string().contains("run task ownership changed"));
16862    }
16863
16864    #[cfg(unix)]
16865    #[tokio::test]
16866    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16867        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16868        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16869        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16870        legacy["status"] = serde_json::json!("voting");
16871        runner.state = serde_json::from_value(legacy).unwrap();
16872        runner.state.save().unwrap();
16873        for _ in 0..3 {
16874            runner.execute_graph().await.unwrap();
16875            assert_eq!(runner.state.status, RunStatus::Blocked);
16876            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16877            assert!(!crate::handover::can_resume(&runner.state));
16878            assert_eq!(
16879                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16880                head
16881            );
16882            assert!(runner.state.reviews.is_empty());
16883            assert!(!runner.state.gate_ran);
16884        }
16885        let notice = crate::notices::run_ended(&runner.state).unwrap();
16886        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16887        runner.state.begin_pass();
16888        runner.state.status = RunStatus::Voting;
16889        runner.deliberate().await.unwrap();
16890        runner.vote().await.unwrap();
16891        runner.finish_after_tally().await.unwrap();
16892        assert_eq!(runner.state.status, RunStatus::Blocked);
16893        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16894        assert_eq!(
16895            crate::notices::run_ended(&runner.state).unwrap().since,
16896            notice.since
16897        );
16898    }
16899
16900    #[cfg(unix)]
16901    #[tokio::test]
16902    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16903        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16904        for detail in [
16905            "fix/retained conflicts with origin/main and did not rebase: conflict",
16906            "rebased fix/retained locally but origin refused the push",
16907            "unknown failure",
16908        ] {
16909            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16910            runner.state.record_base_sync_stop(&head);
16911            assert!(runner.state.authorize_base_sync(&task).is_err());
16912            runner.state.status = RunStatus::Voting;
16913            runner.execute_graph().await.unwrap();
16914            assert_eq!(runner.state.status, RunStatus::Blocked);
16915            assert!(!crate::handover::can_resume(&runner.state));
16916            assert_eq!(
16917                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16918                head
16919            );
16920        }
16921    }
16922
16923    #[cfg(unix)]
16924    #[tokio::test]
16925    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16926        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16927        runner.state.authorize_base_sync(&task).unwrap();
16928        assert!(runner.state.consume_base_recovery(&head));
16929        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16930        runner.state.reserve_base_operation(&head, &tip);
16931        runner.state.save().unwrap();
16932        runner.state = RunState::load(&runner.state.id).unwrap();
16933        assert!(!runner.state.consume_base_recovery(&head));
16934        runner.sync_to_base().await.unwrap();
16935        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16936        assert_eq!(runner.state.base_episode_used(), 1);
16937        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16938        assert!(
16939            runner
16940                .state
16941                .base_sync_recovery
16942                .operations
16943                .last()
16944                .unwrap()
16945                .finished
16946        );
16947        assert!(!runner.state.base_sync_blocked());
16948        runner.sync_to_base().await.unwrap();
16949        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16950    }
16951
16952    #[cfg(unix)]
16953    #[tokio::test]
16954    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
16955        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16956        // Both historical diagnostic spellings retain an exhaustion identity.
16957        let original = runner
16958            .state
16959            .base_sync
16960            .as_ref()
16961            .unwrap()
16962            .conflict
16963            .clone()
16964            .unwrap();
16965        runner.state.base_sync.as_mut().unwrap().conflict = Some(
16966            original
16967                .replace("synchronization(s)", "rebase(s)")
16968                .replace("synchronizing", "rebasing"),
16969        );
16970        runner.state.base_sync_recovery = Default::default();
16971        runner.state.record_base_sync_stop(&head);
16972        assert_eq!(
16973            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16974            crate::base_sync_recovery::StopKind::Exhausted
16975        );
16976        runner.state.authorize_base_sync(&task).unwrap();
16977        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16978        let auth = legacy["base_sync_recovery"]["authorizations"][0]
16979            .as_object_mut()
16980            .unwrap();
16981        auth.remove("interval");
16982        auth.remove("scope");
16983        runner.state = serde_json::from_value(legacy).unwrap();
16984        assert!(runner.state.consume_base_recovery(&head));
16985        assert_eq!(runner.state.base_episode_used(), 3);
16986        runner.sync_to_base().await.unwrap();
16987        assert_eq!(runner.state.base_episode_used(), 4);
16988        runner.state.resource_block = Some(crate::run::ResourceBlock {
16989            node: "gate".into(),
16990            kind: "cache".into(),
16991            detail: "legacy wait".into(),
16992            at: Timestamp::now(),
16993            cause: crate::run::ResourceBlockCause::Unrecorded,
16994        });
16995        runner.state.save().unwrap();
16996        assert!(
16997            runner
16998                .state
16999                .base_sync_recovery
17000                .episodes
17001                .last()
17002                .unwrap()
17003                .boundary
17004                .is_none()
17005        );
17006        runner.state.resource_block = None;
17007        advance_base_fixture(&runner).await;
17008        runner.sync_to_base().await.unwrap();
17009        assert!(runner.state.base_sync_blocked());
17010        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17011    }
17012
17013    #[cfg(unix)]
17014    async fn advance_base_fixture(runner: &Runner) {
17015        let repo = &runner.state.repo;
17016        for args in [
17017            vec!["checkout", "main"],
17018            vec!["commit", "--allow-empty", "-m", "base advanced"],
17019            vec!["push", "origin", "main"],
17020            vec!["checkout", "fix/retained"],
17021        ] {
17022            git::git(repo, &args).await.unwrap();
17023        }
17024    }
17025
17026    #[cfg(unix)]
17027    fn fresh_base_interval_fixture(runner: &mut Runner) {
17028        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17029        runner.state.base_sync.as_mut().unwrap().conflict = None;
17030        runner.state.base_sync_recovery = Default::default();
17031        runner.state.status = RunStatus::Voting;
17032    }
17033
17034    #[cfg(unix)]
17035    #[tokio::test]
17036    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17037        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17038        fresh_base_interval_fixture(&mut runner);
17039        runner.state.begin_pass();
17040        runner.sync_to_base().await.unwrap();
17041        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17042        runner.state.resource_block = Some(crate::run::ResourceBlock {
17043            node: "gate".into(),
17044            kind: "build cache".into(),
17045            detail: "busy".into(),
17046            at: Timestamp::now(),
17047            cause: crate::run::ResourceBlockCause::Timeout {
17048                waited_secs: 1,
17049                limit_secs: 1,
17050            },
17051        });
17052        runner.state.save().unwrap();
17053        let queue = crate::queue::Queue::open();
17054        let mut owner = queue.get(&task).unwrap();
17055        runner.state.task_instruction = Some(owner.instruction.clone());
17056        owner.release_operator("fixture-owner", None);
17057        queue.put(&mut owner).unwrap();
17058        let approval = owner.resumption.clone().unwrap();
17059        runner.state.resource_block = None;
17060        runner.state.save().unwrap();
17061        runner.state = RunState::load(&runner.state.id).unwrap();
17062        advance_base_fixture(&runner).await;
17063        runner.sync_to_base().await.unwrap();
17064        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17065        assert_eq!(runner.state.base_episode_used(), 1);
17066        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17067        assert!(
17068            runner.state.base_sync_recovery.episodes[0]
17069                .boundary
17070                .as_ref()
17071                .unwrap()
17072                .consumed
17073        );
17074        assert!(
17075            !runner
17076                .state
17077                .renew_base_episode(&task, &approval, &head, "base")
17078        );
17079        for _ in 0..3 {
17080            owner.release_operator("fixture-owner", None);
17081            queue.put(&mut owner).unwrap();
17082            runner.state = RunState::load(&runner.state.id).unwrap();
17083            runner.state.begin_pass();
17084            runner.sync_to_base().await.unwrap();
17085            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17086            assert_eq!(runner.state.base_episode_used(), 1);
17087        }
17088        runner.state.resource_block = Some(crate::run::ResourceBlock {
17089            node: "gate".into(),
17090            kind: "disk pressure".into(),
17091            detail: "free space required".into(),
17092            at: Timestamp::now(),
17093            cause: crate::run::ResourceBlockCause::Other,
17094        });
17095        runner.state.end_pass_parked(ParkCause::Interrupt {
17096            reason: "disk pressure".into(),
17097        });
17098        runner.state.save().unwrap();
17099        owner.release_operator("fixture-owner-after-disk", None);
17100        queue.put(&mut owner).unwrap();
17101        runner.state.resource_block = None;
17102        advance_base_fixture(&runner).await;
17103        runner.sync_to_base().await.unwrap();
17104        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17105        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17106        assert_eq!(runner.state.base_episode_used(), 1);
17107        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17108    }
17109
17110    #[cfg(unix)]
17111    #[tokio::test]
17112    async fn base_recovery_continuous_advancement_stops_at_four() {
17113        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17114        fresh_base_interval_fixture(&mut runner);
17115        for spent in 1..=BASE_SYNC_ROUNDS {
17116            runner.sync_to_base().await.unwrap();
17117            assert!(!runner.state.base_sync_blocked());
17118            assert_eq!(runner.state.base_episode_used(), spent);
17119            advance_base_fixture(&runner).await;
17120        }
17121        runner.sync_to_base().await.unwrap();
17122        assert_eq!(
17123            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17124            crate::base_sync_recovery::StopKind::Exhausted
17125        );
17126        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17127        assert!(stop.unverified_sync);
17128        let action = runner.state.base_sync_next_action();
17129        assert!(action.contains("never reached its gate"));
17130        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17131        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17132        runner.state = RunState::load(&runner.state.id).unwrap();
17133        runner.sync_to_base().await.unwrap();
17134        assert_eq!(runner.state.base_episode_used(), 4);
17135        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17136    }
17137
17138    #[cfg(unix)]
17139    #[tokio::test]
17140    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17141        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17142        runner.state.ensure_base_episode();
17143        // Verification progress alone is not evidence of a successful sync.
17144        runner.state.record_base_progress(&head);
17145        runner.state.base_sync_recovery.stop = None;
17146        runner.state.record_base_sync_stop(&head);
17147        assert!(
17148            !runner
17149                .state
17150                .base_sync_recovery
17151                .stop
17152                .as_ref()
17153                .unwrap()
17154                .unverified_sync
17155        );
17156
17157        runner.state.finish_base_operation(Some(&head));
17158        // A gate for another head cannot verify this synchronization.
17159        runner.state.gate_ran = true;
17160        runner.state.gate_head = Some("older-head".into());
17161        runner.state.base_sync_recovery.stop = None;
17162        runner.state.record_base_sync_stop(&head);
17163        runner.state.save().unwrap();
17164        runner.state = RunState::load(&runner.state.id).unwrap();
17165        assert!(
17166            runner
17167                .state
17168                .base_sync_recovery
17169                .stop
17170                .as_ref()
17171                .unwrap()
17172                .unverified_sync
17173        );
17174
17175        runner.state.gate_head = Some(head.clone());
17176        runner.state.base_sync_recovery.stop = None;
17177        runner.state.record_base_sync_stop(&head);
17178        assert!(
17179            !runner
17180                .state
17181                .base_sync_recovery
17182                .stop
17183                .as_ref()
17184                .unwrap()
17185                .unverified_sync
17186        );
17187        assert!(
17188            !runner
17189                .state
17190                .base_sync_next_action()
17191                .contains("never reached its gate")
17192        );
17193
17194        // Historical records cannot establish the missing synchronization evidence.
17195        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17196        legacy["base_sync_recovery"]["stop"]
17197            .as_object_mut()
17198            .unwrap()
17199            .remove("unverified_sync");
17200        legacy["base_sync_recovery"]["episodes"][0]
17201            .as_object_mut()
17202            .unwrap()
17203            .remove("last_sync_head");
17204        runner.state = serde_json::from_value(legacy).unwrap();
17205        assert!(
17206            !runner
17207                .state
17208                .base_sync_recovery
17209                .stop
17210                .as_ref()
17211                .unwrap()
17212                .unverified_sync
17213        );
17214        assert!(
17215            runner.state.base_sync_recovery.episodes[0]
17216                .last_sync_head
17217                .is_none()
17218        );
17219    }
17220
17221    #[cfg(unix)]
17222    #[tokio::test]
17223    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17224        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17225        runner.state.authorize_base_sync(&task).unwrap();
17226        assert!(
17227            runner
17228                .state
17229                .base_sync_next_action()
17230                .contains("Approval recorded")
17231        );
17232        assert!(
17233            runner
17234                .state
17235                .base_sync_next_action()
17236                .contains("4/4 in episode 0, 4 lifetime")
17237        );
17238        assert!(runner.state.consume_base_recovery(&head));
17239        assert!(
17240            runner
17241                .state
17242                .base_sync_next_action()
17243                .contains("0/4 in episode 1, 4 lifetime")
17244        );
17245    }
17246
17247    #[cfg(unix)]
17248    #[tokio::test]
17249    async fn base_recovery_changed_head_invalidates_without_spending() {
17250        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17251        runner.state.authorize_base_sync(&task).unwrap();
17252        git::git(
17253            &runner.state.repo,
17254            &["commit", "--allow-empty", "-m", "external head change"],
17255        )
17256        .await
17257        .unwrap();
17258        runner.sync_to_base().await.unwrap();
17259        let a = runner
17260            .state
17261            .base_sync_recovery
17262            .authorizations
17263            .last()
17264            .unwrap();
17265        assert!(a.consumed);
17266        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17267        runner.state = RunState::load(&runner.state.id).unwrap();
17268        assert!(
17269            runner
17270                .state
17271                .base_sync_next_action()
17272                .contains("Approval invalidated")
17273        );
17274        assert!(!runner.state.base_recovery_pending());
17275        assert!(runner.state.base_sync_blocked());
17276        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17277    }
17278
17279    #[cfg(unix)]
17280    #[tokio::test]
17281    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17282        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17283        runner.state.driver_exited = true;
17284        runner.state.save().unwrap();
17285        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17286            .await
17287            .unwrap();
17288        runner.state = RunState::load(&runner.state.id).unwrap();
17289        advance_base_fixture(&runner).await;
17290        runner.sync_to_base().await.unwrap();
17291        let a = runner
17292            .state
17293            .base_sync_recovery
17294            .authorizations
17295            .last()
17296            .unwrap();
17297        assert!(
17298            a.consumed
17299                && a.invalidated
17300                    .as_deref()
17301                    .unwrap()
17302                    .contains("fetched base changed")
17303        );
17304        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17305        assert_eq!(
17306            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17307            head
17308        );
17309        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17310            .await
17311            .unwrap();
17312        runner.state = RunState::load(&runner.state.id).unwrap();
17313        runner.sync_to_base().await.unwrap();
17314        assert!(!runner.state.base_sync_blocked());
17315        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17316    }
17317
17318    #[cfg(unix)]
17319    #[tokio::test]
17320    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17321        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17322        runner.state.authorize_base_sync(&task).unwrap();
17323        assert!(runner.state.consume_base_recovery(&head));
17324        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17325        runner.state.reserve_base_operation(&head, &tip);
17326        runner.state.save().unwrap();
17327        let scratch = runner.state.dir().join("base-sync");
17328        assert_eq!(
17329            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17330                .await
17331                .unwrap(),
17332            crate::rebase::Rebased::Applied
17333        );
17334        runner.state = RunState::load(&runner.state.id).unwrap();
17335        runner.sync_to_base().await.unwrap();
17336        assert!(
17337            runner
17338                .state
17339                .base_sync_recovery
17340                .operations
17341                .last()
17342                .unwrap()
17343                .finished
17344        );
17345        assert_eq!(runner.state.base_episode_used(), 1);
17346        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17347        assert!(!runner.state.base_sync_blocked());
17348    }
17349
17350    #[cfg(unix)]
17351    #[tokio::test]
17352    async fn base_recovery_crash_preserves_untracked_files() {
17353        for ignored in [false, true] {
17354            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17355            runner.state.authorize_base_sync(&task).unwrap();
17356            assert!(runner.state.consume_base_recovery(&head));
17357            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17358            runner.state.reserve_base_operation(&head, &tip);
17359            runner.state.save().unwrap();
17360            let scratch = runner.state.dir().join("base-sync");
17361            assert_eq!(
17362                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17363                    .await
17364                    .unwrap(),
17365                crate::rebase::Rebased::Applied
17366            );
17367            let repo = &runner.state.repo;
17368            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17369            assert_ne!(result, head);
17370            if ignored {
17371                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17372                    .await
17373                    .unwrap();
17374                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17375            }
17376            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17377            assert!(
17378                git::git_raw(repo, &["diff", "--quiet", &head])
17379                    .await
17380                    .unwrap()
17381                    .ok()
17382            );
17383            // Resume after the branch moved, before its checkout was reset.
17384            runner.state = RunState::load(&runner.state.id).unwrap();
17385            runner.sync_to_base().await.unwrap();
17386            assert!(runner.state.base_sync_blocked());
17387            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17388            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17389            assert_eq!(
17390                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17391                result
17392            );
17393            assert_eq!(
17394                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17395                "retained work\n"
17396            );
17397            assert!(
17398                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17399                    .await
17400                    .unwrap()
17401                    .ok()
17402            );
17403        }
17404    }
17405
17406    #[cfg(unix)]
17407    #[tokio::test]
17408    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17409        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17410        runner.state.authorize_base_sync(&task).unwrap();
17411        assert!(runner.state.consume_base_recovery(&head));
17412        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17413        runner.state.reserve_base_operation(&head, &tip);
17414        runner.state.save().unwrap();
17415        let scratch = runner.state.dir().join("base-sync");
17416        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17417            .await
17418            .unwrap();
17419        git::sync_to_head(&runner.state.repo).await.unwrap();
17420        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17421        git::git(&runner.state.repo, &["add", "foreign"])
17422            .await
17423            .unwrap();
17424        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17425            .await
17426            .unwrap();
17427        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17428        runner.state = RunState::load(&runner.state.id).unwrap();
17429        runner.sync_to_base().await.unwrap();
17430        assert!(runner.state.base_sync_blocked());
17431        assert!(
17432            !runner
17433                .state
17434                .base_sync_recovery
17435                .operations
17436                .last()
17437                .unwrap()
17438                .finished
17439        );
17440        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17441        assert_eq!(
17442            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17443            foreign_head
17444        );
17445    }
17446
17447    #[cfg(unix)]
17448    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17449    {
17450        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17451        fresh_base_interval_fixture(&mut runner);
17452        let repo = &runner.state.repo;
17453        git::git(repo, &["checkout", "main"]).await.unwrap();
17454        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17455        git::git(repo, &["add", "retained"]).await.unwrap();
17456        git::git(repo, &["commit", "-m", "upstream retained change"])
17457            .await
17458            .unwrap();
17459        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17460        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17461        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17462        let scratch = runner.state.dir().join("base-sync");
17463        assert!(matches!(
17464            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17465                .await
17466                .unwrap(),
17467            git::RebaseStart::Conflicted(_)
17468        ));
17469        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17470            agent: "fixture".into(),
17471            paths: vec!["retained".into()],
17472            from: Some(head.clone()),
17473            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17474            index_outside: None,
17475            finished: false,
17476            error: None,
17477            launcher: None,
17478            infra: None,
17479            outage: None,
17480        });
17481        (tmp, runner, head, tip, scratch)
17482    }
17483
17484    #[cfg(unix)]
17485    #[tokio::test]
17486    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17487        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17488        // A resource stop from before reservations existed, with resolved
17489        // contents left for the launcher's ordinary staging continuation.
17490        std::fs::write(
17491            scratch.join("retained"),
17492            "upstream and retained resolution\n",
17493        )
17494        .unwrap();
17495        runner.state.resource_block = Some(crate::run::ResourceBlock {
17496            node: "rebase".into(),
17497            kind: "repository coordination".into(),
17498            detail: "busy".into(),
17499            at: Timestamp::now(),
17500            cause: crate::run::ResourceBlockCause::Other,
17501        });
17502        runner.state.save().unwrap();
17503        // Advance the remote without moving either checkout participating
17504        // in the standing rebase.
17505        let newer = git::git(
17506            &runner.state.repo,
17507            &[
17508                "commit-tree",
17509                "main^{tree}",
17510                "-p",
17511                "main",
17512                "-m",
17513                "base advanced during wait",
17514            ],
17515        )
17516        .await
17517        .unwrap();
17518        git::git(
17519            &runner.state.repo,
17520            &[
17521                "push",
17522                "origin",
17523                &format!("{}:refs/heads/main", newer.trim()),
17524            ],
17525        )
17526        .await
17527        .unwrap();
17528        runner.state = RunState::load(&runner.state.id).unwrap();
17529        runner.state.resource_block = None;
17530        runner.sync_to_base().await.unwrap();
17531        assert!(!runner.state.base_sync_blocked());
17532        let op = &runner.state.base_sync_recovery.operations[0];
17533        assert_eq!(op.head, head);
17534        assert_eq!(op.tip, tip);
17535        assert!(op.finished);
17536        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17537        assert_eq!(runner.state.rebase_fixes.len(), 1);
17538        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17539            .await
17540            .unwrap();
17541        assert_ne!(latest, tip);
17542        runner.state = RunState::load(&runner.state.id).unwrap();
17543        runner.sync_to_base().await.unwrap();
17544        assert!(!runner.state.base_sync_blocked());
17545        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17546        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17547        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17548    }
17549
17550    #[cfg(unix)]
17551    #[tokio::test]
17552    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17553        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17554        runner.state.reserve_base_operation(&head, &tip);
17555        runner.state.save().unwrap();
17556        std::fs::write(
17557            scratch.join("retained"),
17558            "upstream and retained resolution\n",
17559        )
17560        .unwrap();
17561        git::git(&scratch, &["add", "retained"]).await.unwrap();
17562        git::git(
17563            &scratch,
17564            &["-c", "core.editor=true", "rebase", "--continue"],
17565        )
17566        .await
17567        .unwrap();
17568        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17569        assert!(
17570            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17571                .await
17572                .unwrap()
17573                .is_empty()
17574        );
17575        assert_eq!(
17576            crate::rebase::finish(
17577                &mut runner.state,
17578                &scratch,
17579                "fix/retained",
17580                &head,
17581                &tip,
17582                &["retained".into()],
17583                ""
17584            )
17585            .await
17586            .unwrap(),
17587            crate::rebase::Rebased::Applied
17588        );
17589        assert!(!scratch.exists());
17590        // Simulate losing the caller before finish_base_operation/save.
17591        runner.state = RunState::load(&runner.state.id).unwrap();
17592        let op = &runner.state.base_sync_recovery.operations[0];
17593        assert!(!op.finished);
17594        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17595        runner.sync_to_base().await.unwrap();
17596        assert!(!runner.state.base_sync_blocked());
17597        assert!(runner.state.base_sync_recovery.operations[0].finished);
17598        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17599        assert_eq!(runner.state.rebase_fixes.len(), 1);
17600        let winner = runner.state.winner().unwrap().clone();
17601        let wait = runner.repository_wait();
17602        assert!(
17603            verify_current_head(&mut runner.state, &winner, &wait)
17604                .await
17605                .unwrap()
17606        );
17607        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17608    }
17609
17610    #[cfg(unix)]
17611    #[tokio::test]
17612    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17613        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17614            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17615            runner.state.reserve_base_operation(&head, &tip);
17616            runner.state.save().unwrap();
17617            std::fs::write(
17618                scratch.join("retained"),
17619                "upstream and retained resolution\n",
17620            )
17621            .unwrap();
17622            git::git(&scratch, &["add", "retained"]).await.unwrap();
17623            git::git(
17624                &scratch,
17625                &["-c", "core.editor=true", "rebase", "--continue"],
17626            )
17627            .await
17628            .unwrap();
17629            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17630            assert!(!git::rebase_in_progress(&scratch).await);
17631            assert!(
17632                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17633                    .await
17634                    .unwrap()
17635                    .is_empty()
17636            );
17637            match damage {
17638                Some("record") => {
17639                    let path =
17640                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17641                            .await
17642                            .unwrap();
17643                    std::fs::write(scratch.join(path), "").unwrap();
17644                }
17645                Some("foreign") => {
17646                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17647                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17648                    git::git(&scratch, &["commit", "-m", "foreign work"])
17649                        .await
17650                        .unwrap();
17651                }
17652                Some("dirty") => {
17653                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17654                }
17655                _ => {}
17656            }
17657            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17658            // Crash immediately after Git continuation, before finish runs.
17659            runner.state = RunState::load(&runner.state.id).unwrap();
17660            assert!(
17661                runner.state.base_sync_recovery.operations[0]
17662                    .validated_head
17663                    .is_none()
17664            );
17665            runner.sync_to_base().await.unwrap();
17666            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17667            assert_eq!(runner.state.base_episode_used(), 1);
17668            assert_eq!(runner.state.rebase_fixes.len(), 1);
17669            assert_eq!(
17670                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17671                retained_head
17672            );
17673            if damage.is_some() {
17674                assert!(runner.state.base_sync_blocked());
17675                assert!(scratch.exists());
17676                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17677                assert!(
17678                    runner.state.base_sync_recovery.operations[0]
17679                        .validated_head
17680                        .is_none()
17681                );
17682            } else {
17683                assert!(!runner.state.base_sync_blocked());
17684                assert!(!scratch.exists());
17685                assert!(runner.state.base_sync_recovery.operations[0].finished);
17686                assert_eq!(
17687                    runner.state.base_sync_recovery.operations[0]
17688                        .validated_head
17689                        .as_deref(),
17690                    Some(result.as_str())
17691                );
17692                let winner = runner.state.winner().unwrap().clone();
17693                let wait = runner.repository_wait();
17694                assert!(
17695                    verify_current_head(&mut runner.state, &winner, &wait)
17696                        .await
17697                        .unwrap()
17698                );
17699                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17700                runner.state = RunState::load(&runner.state.id).unwrap();
17701                runner.sync_to_base().await.unwrap();
17702                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17703            }
17704        }
17705    }
17706
17707    #[cfg(unix)]
17708    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17709        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17710        // This is an ordinary gate stop, not an authorized base-budget recovery.
17711        runner.state.prep_complete = Some(true);
17712        runner.state.base_sync = None;
17713        runner.state.base_sync_recovery = Default::default();
17714        runner.state.config.graph.gate_fix_rounds = 1;
17715        let mut review = review_round(true, 0, 1, 1, false, true);
17716        review.head = head.clone();
17717        review.verified_head = Some(head.clone());
17718        runner.state.reviews = vec![review];
17719        runner.state.gate = vec![CommandOutcome {
17720            command: "old gate".into(),
17721            code: Some(1),
17722            output_tail: "old failure".into(),
17723            duration_ms: 0,
17724            resource_blocked: false,
17725        }];
17726        runner.state.gate_head = Some(head.clone());
17727        runner.state.gate_ran = true;
17728        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17729            agent: "fixture".into(),
17730            failed: runner.state.gate.clone(),
17731            notes: "spent".into(),
17732            committed: false,
17733            error: None,
17734        }];
17735        runner.state.land_approval = Some(crate::run::LandApproval {
17736            question: "old approval".into(),
17737            head: head.clone(),
17738        });
17739        runner.state.status = RunStatus::Blocked;
17740        (tmp, runner, head)
17741    }
17742
17743    #[cfg(unix)]
17744    #[tokio::test]
17745    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17746        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17747            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17748            let phases = tmp.path().join("phases");
17749            runner.state.config.verify.e2e = vec![format!(
17750                "echo e2e >> '{}'; {}",
17751                phases.display(),
17752                if e2e_ok {
17753                    "true"
17754                } else {
17755                    "echo new-e2e-failure; false"
17756                }
17757            )];
17758            runner.state.config.verify.gate = vec![format!(
17759                "echo gate >> '{}'; {}",
17760                phases.display(),
17761                if gate_ok {
17762                    "true"
17763                } else {
17764                    "echo new-gate-failure; false"
17765                }
17766            )];
17767            runner.state.save().unwrap();
17768            runner.state = RunState::load(&runner.state.id).unwrap();
17769            runner.sync_to_base().await.unwrap();
17770            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17771            assert_ne!(head, old);
17772            // Reentry after Git updated but before verification was invalidated.
17773            runner.state = RunState::load(&runner.state.id).unwrap();
17774            runner.execute_graph().await.unwrap();
17775            assert_eq!(runner.state.gate_fixes.len(), 1);
17776            assert_eq!(runner.state.reviews.len(), 1);
17777            assert_eq!(runner.state.reviews[0].head, old);
17778            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17779            assert_eq!(runner.state.verification_history.len(), 1);
17780            let history = &runner.state.verification_history[0];
17781            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17782            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17783            assert!(history.archived_at.is_some());
17784            assert_eq!(history.fixes_spent, Some(1));
17785            assert_eq!(history.gate[0].output_tail, "old failure");
17786            let archive_events: Vec<_> = runner
17787                .state
17788                .events
17789                .iter()
17790                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17791                .collect();
17792            assert_eq!(archive_events.len(), 1);
17793            assert!(archive_events[0].message.contains(&short(&old)));
17794            assert!(archive_events[0].message.contains(&short(&head)));
17795            assert!(
17796                archive_events[0]
17797                    .message
17798                    .contains("1 gate-fix round(s) already spent stay spent")
17799            );
17800            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17801            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17802            if e2e_ok {
17803                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17804                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17805            } else {
17806                assert!(!runner.state.gate_ran);
17807            }
17808            assert_eq!(
17809                runner.state.status,
17810                if e2e_ok && gate_ok {
17811                    RunStatus::Ready
17812                } else {
17813                    RunStatus::Blocked
17814                }
17815            );
17816            if !e2e_ok || !gate_ok {
17817                let cause = runner.state.verification_blocking_cause().unwrap();
17818                assert!(cause.contains(&head));
17819                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17820                // A later successful progress event must not become the cause.
17821                runner.state.event("land", "rebased successfully");
17822                let notice = crate::notices::run_ended(&runner.state).unwrap();
17823                assert!(!format!("{notice:?}").contains("rebased successfully"));
17824                assert!(runner.state.merge.is_none());
17825            }
17826            let before = std::fs::read_to_string(&phases).unwrap();
17827            runner.state.save().unwrap();
17828            runner.state = RunState::load(&runner.state.id).unwrap();
17829            runner.review_loop().await.unwrap();
17830            runner.gate().await.unwrap();
17831            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17832            assert_eq!(runner.state.verification_history.len(), 1);
17833            let history = &runner.state.verification_history[0];
17834            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17835            assert!(history.archived_at.is_some());
17836            assert_eq!(history.fixes_spent, Some(1));
17837        }
17838    }
17839
17840    #[cfg(unix)]
17841    #[tokio::test]
17842    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17843        let (tmp, mut runner, head) = resumed_gate_fixture().await;
17844        let marker = tmp.path().join("gate-ran");
17845        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17846        runner.review_loop().await.unwrap();
17847        runner.gate().await.unwrap();
17848        assert_eq!(runner.state.status, RunStatus::Blocked);
17849        assert!(!marker.exists());
17850        assert!(runner.state.verification_history.is_empty());
17851        runner.state.gate_head = None;
17852        runner.gate().await.unwrap();
17853        assert!(marker.exists());
17854        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17855        assert_eq!(runner.state.verification_history.len(), 1);
17856        assert_eq!(runner.state.gate_fixes.len(), 1);
17857    }
17858
17859    #[cfg(unix)]
17860    #[tokio::test]
17861    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17862        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17863        runner.sync_to_base().await.unwrap();
17864        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17865        runner.state.reconcile_verification_head(&head);
17866        // Both crash-before-command and resource wait are unresolved evidence.
17867        for outcomes in [
17868            Vec::new(),
17869            vec![CommandOutcome {
17870                command: "true".into(),
17871                code: None,
17872                output_tail: "cache unavailable".into(),
17873                duration_ms: 0,
17874                resource_blocked: true,
17875            }],
17876        ] {
17877            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17878                head: head.clone(),
17879                passed: false,
17880                outcomes,
17881            });
17882            runner.state.gate_ran = false;
17883            runner.state.gate.clear();
17884            runner.state.save().unwrap();
17885            runner.state = RunState::load(&runner.state.id).unwrap();
17886            runner.merge().await.unwrap();
17887            assert!(runner.state.merge.is_none());
17888            runner.review_loop().await.unwrap();
17889            runner.gate().await.unwrap();
17890            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17891            assert!(runner.state.gate_allows_progress());
17892            assert_eq!(runner.state.gate_fixes.len(), 1);
17893            assert!(runner.state.merge.is_none());
17894        }
17895        // Moving the winner after a successful gate cannot publish an unchecked SHA.
17896        git::git(
17897            &runner.state.repo,
17898            &["commit", "--allow-empty", "-m", "external update"],
17899        )
17900        .await
17901        .unwrap();
17902        runner.merge().await.unwrap();
17903        assert!(runner.state.merge.is_none());
17904        assert_eq!(runner.state.status, RunStatus::Blocked);
17905    }
17906
17907    #[cfg(unix)]
17908    #[tokio::test]
17909    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17910        for waiting_on_gate in [false, true] {
17911            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17912            runner.sync_to_base().await.unwrap();
17913            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17914            runner.state.reconcile_verification_head(&head);
17915            if waiting_on_gate {
17916                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17917                    head: head.clone(),
17918                    passed: true,
17919                    outcomes: Vec::new(),
17920                });
17921            }
17922            let cache = tmp.path().join("cache");
17923            runner.state.config.verify.e2e =
17924                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17925            runner.state.config.verify.gate = vec!["true".into()];
17926            runner.state.config.graph.timeout_resource_wait = 1;
17927            let owner =
17928                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17929            let guard =
17930                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17931                    crate::cache::AcquireOutcome::Acquired(g) => g,
17932                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17933                };
17934            runner.review_loop().await.unwrap();
17935            runner.gate().await.unwrap();
17936            assert!(!runner.state.gate_ran);
17937            assert_eq!(runner.state.gate_fixes.len(), 1);
17938            assert!(runner.state.resource_block.is_some());
17939            runner.merge().await.unwrap();
17940            assert!(runner.state.merge.is_none());
17941            drop(guard);
17942            runner.state = RunState::load(&runner.state.id).unwrap();
17943            // The driver clears the previous pass's resource stop on resume.
17944            runner.state.resource_block = None;
17945            runner.review_loop().await.unwrap();
17946            runner.gate().await.unwrap();
17947            assert!(runner.state.gate_allows_progress());
17948            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17949            assert_eq!(runner.state.gate_fixes.len(), 1);
17950        }
17951    }
17952
17953    #[cfg(unix)]
17954    #[tokio::test]
17955    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
17956        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17957        runner.sync_to_base().await.unwrap();
17958        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17959        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
17960        // Verification passes before pre_gate and fails on the mechanically changed head.
17961        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
17962        runner.review_loop().await.unwrap();
17963        runner.gate().await.unwrap();
17964        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17965        assert_ne!(before, after);
17966        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
17967        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
17968        assert!(!runner.state.gate_allows_progress());
17969        runner.merge().await.unwrap();
17970        assert!(runner.state.merge.is_none());
17971        assert!(
17972            runner
17973                .state
17974                .verification_history
17975                .iter()
17976                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
17977        );
17978
17979        // An incomplete review is still an independent hold on a changed head.
17980        runner.state.reviews[0].clean = false;
17981        runner.state.reviews[0].answered = 0;
17982        runner.state.config.verify.e2e = vec!["true".into()];
17983        runner.review_loop().await.unwrap();
17984        runner.gate().await.unwrap();
17985        runner.merge().await.unwrap();
17986        assert_eq!(runner.state.status, RunStatus::Blocked);
17987        assert!(runner.state.merge.is_none());
17988        assert!(
17989            runner
17990                .state
17991                .verification_blocking_cause()
17992                .unwrap()
17993                .contains("review blocked")
17994        );
17995    }
17996
17997    #[cfg(unix)]
17998    #[tokio::test]
17999    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18000        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18001        runner.state.gate_head = None;
18002        runner.state.config.verify.gate =
18003            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18004        runner.review_loop().await.unwrap();
18005        runner.gate().await.unwrap();
18006        assert_ne!(
18007            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18008            old
18009        );
18010        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18011        assert!(!runner.state.gate_allows_progress());
18012        assert_eq!(runner.state.status, RunStatus::Blocked);
18013        runner.merge().await.unwrap();
18014        assert!(runner.state.merge.is_none());
18015    }
18016
18017    #[cfg(unix)]
18018    #[tokio::test]
18019    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18020        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18021        let mut old_review = review_round(true, 0, 1, 1, false, true);
18022        old_review.head = head.clone();
18023        old_review.verified_head = Some(head.clone());
18024        runner.state.reviews = vec![old_review];
18025        runner.state.gate = vec![CommandOutcome {
18026            command: "old gate".into(),
18027            code: None,
18028            output_tail: "timed out after 2400s".into(),
18029            duration_ms: 2_400_000,
18030            resource_blocked: false,
18031        }];
18032        runner.state.gate_ran = true;
18033        runner.state.gate_head = Some(head.clone());
18034        runner.state.land_approval = Some(crate::run::LandApproval {
18035            question: "old-head-approval".into(),
18036            head: head.clone(),
18037        });
18038        runner.state.authorize_base_sync(&task).unwrap();
18039        runner.sync_to_base().await.unwrap();
18040        assert!(!runner.state.base_sync_blocked());
18041        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18042        assert_ne!(head, fresh);
18043        assert_eq!(runner.state.reviews.len(), 1);
18044        assert!(!runner.state.gate_ran);
18045        let saved = &runner.state.base_sync_recovery.authorizations[0];
18046        assert_eq!(saved.reviews[0].head, head);
18047        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18048        assert!(saved.gate_ran);
18049        assert!(!saved.gate[0].ok());
18050        runner.review_loop().await.unwrap();
18051        runner.gate().await.unwrap();
18052        assert_eq!(runner.state.reviews[0].head, head);
18053        assert_eq!(
18054            runner.state.reviews[0].verified_head.as_deref(),
18055            Some(head.as_str())
18056        );
18057        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18058        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18059        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18060        assert!(runner.state.gate_ran);
18061        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18062        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18063        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18064        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18065        assert_eq!(
18066            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18067            "retained change"
18068        );
18069    }
18070
18071    #[cfg(unix)]
18072    #[tokio::test]
18073    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18074        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18075        let queue = crate::queue::Queue::open();
18076        let mut owner = queue.get(&task).unwrap();
18077        owner.release_operator("test-resume", None);
18078        queue.put(&mut owner).unwrap();
18079        let generation = owner.resumption.as_ref().unwrap().generation;
18080        runner.state.task_instruction = Some(owner.instruction.clone());
18081        runner.state.driver_exited = true;
18082        runner.state.save().unwrap();
18083        let home = crate::run::home();
18084        let _claim = queue.claim(&task).unwrap();
18085        owner.resumption.as_mut().unwrap().at =
18086            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18087        queue.put(&mut owner).unwrap();
18088        assert!(
18089            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18090                .await
18091                .is_err(),
18092            "an approval from before the stop grants nothing"
18093        );
18094        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18095        queue.put(&mut owner).unwrap();
18096        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18097            .await
18098            .unwrap();
18099        assert!(
18100            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18101                .await
18102                .is_err()
18103        );
18104        runner.state = RunState::load(&runner.state.id).unwrap();
18105        assert_eq!(
18106            runner.state.base_sync_recovery.authorizations[0].generation,
18107            Some(generation)
18108        );
18109        runner.sync_to_base().await.unwrap();
18110        let repo = runner.state.repo.clone();
18111        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18112        git::git(&repo, &["checkout", "main"]).await.unwrap();
18113        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18114        git::git(&repo, &["add", "."]).await.unwrap();
18115        git::git(&repo, &["commit", "-m", "move base again"])
18116            .await
18117            .unwrap();
18118        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18119        git::git(&repo, &["checkout", "fix/retained"])
18120            .await
18121            .unwrap();
18122        runner.sync_to_base().await.unwrap();
18123        assert!(!runner.state.base_sync_blocked());
18124        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18125        assert_eq!(runner.state.base_episode_used(), 2);
18126        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18127        assert_ne!(fresh, head);
18128        assert!(!runner.state.base_recovery_pending());
18129        // Even replaying the generation with a later timestamp grants nothing.
18130        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18131        queue.put(&mut owner).unwrap();
18132        assert!(
18133            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18134                .await
18135                .is_err()
18136        );
18137        runner.state = RunState::load(&runner.state.id).unwrap();
18138        runner.sync_to_base().await.unwrap();
18139        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18140        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18141        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18142    }
18143
18144    #[cfg(unix)]
18145    #[tokio::test]
18146    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18147        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18148        runner.state.base_sync = None;
18149        runner.state.status = RunStatus::Blocked;
18150        let mut second = runner.state.candidates[0].clone();
18151        second.label = 'B';
18152        runner.state.candidates.push(second);
18153        assert!(runner.state.votes.is_empty());
18154        assert!(runner.state.deliberation.is_empty());
18155        runner.deliberate().await.unwrap();
18156        runner.vote().await.unwrap();
18157        assert_eq!(runner.state.status, RunStatus::Blocked);
18158        assert!(runner.state.votes.is_empty());
18159        assert!(runner.state.deliberation.is_empty());
18160    }
18161
18162    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18163    /// reads nothing else off it.
18164    fn runner_at(status: RunStatus) -> Runner {
18165        let mut state = RunState::new(
18166            PathBuf::from("/nonexistent/repo"),
18167            "main".to_owned(),
18168            "deadbeef".to_owned(),
18169            "task".to_owned(),
18170            Config::default(),
18171        );
18172        // These fixtures test execution flow, independent of host disk capacity.
18173        state.config.disk.min_free_bytes = 0;
18174        // These fixtures share one home and save concurrently. Short clock
18175        // IDs can collide, making an ownership test read another test's run.
18176        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18177        state.id = format!(
18178            "20000101-000000-{:04x}",
18179            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18180        );
18181        state.status = status;
18182        Runner {
18183            state,
18184            roles: ResolvedRoles {
18185                implementers: Vec::new(),
18186                judges: Vec::new(),
18187                reviewers: Vec::new(),
18188                fixer: None,
18189                conductor: conductor(),
18190                implementer_roster: Vec::new(),
18191                judge_roster: Vec::new(),
18192                reviewer_roster: Vec::new(),
18193            },
18194            sem: Arc::new(Semaphore::new(1)),
18195            pause: Pause::new(),
18196            interrupt: Pause::new(),
18197        }
18198    }
18199
18200    #[tokio::test]
18201    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18202        crate::run::pin_test_home();
18203        let mut runner = runner_at(RunStatus::Gating);
18204        runner.state.save().unwrap();
18205        let id = runner.state.id.clone();
18206        let passes = runner.state.passes.len();
18207        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18208        let mut claiming = Box::pin(runner.claim_driver());
18209        std::future::poll_fn(|cx| {
18210            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18211            std::task::Poll::Ready(())
18212        })
18213        .await;
18214        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18215        drop(owner);
18216        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18217            .await
18218            .unwrap()
18219            .unwrap();
18220        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18221        assert!(execution.runner.state.resource_block.is_none());
18222        assert_eq!(
18223            execution.runner.state.liveness(false),
18224            crate::run::Liveness::Live
18225        );
18226        drop(execution);
18227        assert!(RunState::load(&id).unwrap().driver_exited);
18228    }
18229
18230    #[tokio::test]
18231    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18232        crate::run::pin_test_home();
18233        let mut runner = runner_at(RunStatus::Gating);
18234        runner.state.save().unwrap();
18235        // run.json can commit, but the subsequent lease metadata write fails.
18236        // Cleanup must run after releasing the short ownership lock.
18237        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18238        assert!(runner.claim_driver().await.is_err());
18239        assert!(runner.state.driver_exited);
18240        assert_eq!(runner.state.driver_pid, None);
18241        let disk = RunState::load(&runner.state.id).unwrap();
18242        assert!(disk.driver_exited);
18243        assert_eq!(disk.driver_pid, None);
18244        assert_eq!(disk.status, RunStatus::Gating);
18245    }
18246
18247    #[tokio::test]
18248    async fn an_early_graph_error_releases_its_driver() {
18249        crate::run::pin_test_home();
18250        let mut runner = runner_at(RunStatus::Blocked);
18251        runner.state.config.graph.selected_implementer = Some("selected".into());
18252        runner.state.save().unwrap();
18253        assert!(runner.execute().await.is_err());
18254        let disk = RunState::load(&runner.state.id).unwrap();
18255        assert!(disk.driver_exited);
18256        assert_eq!(disk.driver_pid, None);
18257        assert_eq!(disk.status, RunStatus::Blocked);
18258    }
18259
18260    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18261        crate::run::MergeRepair {
18262            question: "repair-question".into(),
18263            head: "old-head".into(),
18264            pr: "https://github.com/o/r/pull/42".into(),
18265            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18266            gate: vec![outcome(Some(0), false)],
18267            gate_head: Some("old-head".into()),
18268            gate_fixes: Vec::new(),
18269            pre_gate: Vec::new(),
18270            phase,
18271            failure: None,
18272        }
18273    }
18274
18275    #[tokio::test]
18276    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18277        crate::run::pin_test_home();
18278        let mut runner = runner_at(RunStatus::Landing);
18279        runner.state.merge_repairs.push(merge_repair_request(
18280            crate::run::MergeRepairPhase::Requested,
18281        ));
18282        runner.state.save().unwrap();
18283        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18284        assert!(
18285            runner
18286                .execute()
18287                .await
18288                .unwrap_err()
18289                .to_string()
18290                .contains("already running")
18291        );
18292        assert_eq!(
18293            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18294            crate::run::MergeRepairPhase::Requested
18295        );
18296        drop(claim);
18297        let mut disk = runner.state.clone();
18298        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18299        disk.save().unwrap();
18300        assert!(
18301            runner
18302                .execute()
18303                .await
18304                .unwrap_err()
18305                .to_string()
18306                .contains("decision changed")
18307        );
18308        assert_eq!(
18309            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18310            crate::run::MergeRepairPhase::Complete
18311        );
18312    }
18313
18314    #[test]
18315    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18316        crate::run::pin_test_home();
18317        let mut runner = runner_at(RunStatus::Landing);
18318        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18319        runner.state.reviews = request.reviews.clone();
18320        runner.state.gate = request.gate.clone();
18321        runner.state.gate_ran = true;
18322        runner.state.land_approval = Some(crate::run::LandApproval {
18323            question: "old".into(),
18324            head: "old-head".into(),
18325        });
18326        runner.state.merge_repairs.push(request);
18327        runner.begin_merge_repair_review(0).unwrap();
18328        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18329        assert!(!runner.state.gate_status().ok());
18330        assert!(runner.state.land_approval.is_none());
18331        let disk = RunState::load(&runner.state.id).unwrap();
18332        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18333        assert!(disk.merge_repairs[0].gate[0].ok());
18334        assert_eq!(
18335            disk.merge_repairs[0].phase,
18336            crate::run::MergeRepairPhase::Reviewing
18337        );
18338        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18339        assert!(disk.reviews.is_empty());
18340    }
18341
18342    #[tokio::test]
18343    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18344        crate::run::pin_test_home();
18345        for phase in [
18346            crate::run::MergeRepairPhase::Fixing,
18347            crate::run::MergeRepairPhase::Requested,
18348        ] {
18349            let mut runner = runner_at(RunStatus::Landing);
18350            runner.state.config.graph.land_rounds = 0;
18351            runner.state.merge_repairs.push(merge_repair_request(phase));
18352            runner.state.save().unwrap();
18353            assert!(!runner.repair_merge_review().await.unwrap());
18354            assert_eq!(runner.state.status, RunStatus::Blocked);
18355            assert_eq!(
18356                runner.state.merge_repairs[0].phase,
18357                crate::run::MergeRepairPhase::Failed
18358            );
18359            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18360            assert!(
18361                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18362                    "interrupted"
18363                } else {
18364                    "budget exhausted"
18365                })
18366            );
18367            assert!(runner.state.merge.is_none());
18368        }
18369    }
18370
18371    /// `park_here` folding in the reason `Pause::park_because` recorded -
18372    /// this is what lets an operator reading a run's events tell an
18373    /// interrupt-driven park from an ordinary shutdown park.
18374    #[test]
18375    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18376        crate::run::pin_test_home();
18377        let mut runner = runner_at(RunStatus::Implementing);
18378        let interrupt = Pause::new();
18379        runner.watch_interrupt(interrupt.clone());
18380
18381        interrupt.park_because("task a1b2 asked to run first");
18382
18383        assert!(runner.park_here().expect("park_here"));
18384        assert!(runner.state.parked);
18385        let last = runner.state.events.last().expect("a park event");
18386        assert_eq!(last.node, "park");
18387        assert!(
18388            last.message.contains("task a1b2 asked to run first"),
18389            "expected the interrupt reason in {:?}",
18390            last.message
18391        );
18392    }
18393
18394    /// The cause is stored on the pass when it parks, and the first cause a
18395    /// shared pause was given is the one that stays.
18396    #[test]
18397    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18398        crate::run::pin_test_home();
18399        let mut runner = runner_at(RunStatus::Implementing);
18400        runner.state.begin_pass();
18401        let shutdown = Pause::new();
18402        runner.on_pause(shutdown.clone());
18403        let upgrade = ParkCause::Upgrade {
18404            from: "0.121.0".to_owned(),
18405            to: Some("0.121.1".to_owned()),
18406        };
18407        shutdown.park_for(upgrade.clone());
18408        shutdown.park_for(ParkCause::Operator);
18409
18410        assert!(runner.park_here().expect("park_here"));
18411        assert_eq!(
18412            runner.state.passes[0].end,
18413            Some(crate::run::PassEnd::Parked { cause: upgrade })
18414        );
18415        let last = runner.state.events.last().expect("a park event");
18416        assert!(last.message.contains("0.121.0"), "{}", last.message);
18417    }
18418
18419    #[tokio::test]
18420    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18421        crate::run::pin_test_home();
18422        let mut runner = runner_at(RunStatus::Reviewing);
18423        runner.state.save().unwrap();
18424        let mut execution = runner.claim_driver().await.expect("first execution");
18425        assert_eq!(execution.runner.state.passes.len(), 1);
18426        execution
18427            .driver
18428            .finish(&mut execution.runner.state)
18429            .await
18430            .unwrap();
18431    }
18432
18433    #[tokio::test]
18434    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18435        crate::run::pin_test_home();
18436        let mut runner = runner_at(RunStatus::Reviewing);
18437        runner.state.config.agents = vec![conductor()];
18438        runner.state.config.graph.timeout_resource_wait = 0;
18439        runner.state.save().unwrap();
18440        let id = runner.state.id.clone();
18441        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18442        assert!(runner.claim_driver().await.is_err());
18443        drop(owner);
18444        drop(runner);
18445        let mut resumed = Runner::resume(&id).unwrap();
18446        assert_eq!(resumed.state.driver_claimed, Some(false));
18447        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18448        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18449        let mut execution = resumed
18450            .claim_driver()
18451            .await
18452            .expect("retry initial claim after reload");
18453        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18454        assert_eq!(execution.runner.state.passes.len(), 1);
18455        execution
18456            .driver
18457            .finish(&mut execution.runner.state)
18458            .await
18459            .unwrap();
18460    }
18461
18462    #[tokio::test]
18463    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18464        crate::run::pin_test_home();
18465        let mut runner = runner_at(RunStatus::Reviewing);
18466        runner.state.config.agents = vec![conductor()];
18467        runner.state.save().unwrap();
18468        let id = runner.state.id.clone();
18469        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18470        let mut resumed = Runner::resume(&id).unwrap();
18471        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18472        assert!(resumed.claim_driver().await.is_err());
18473        assert!(RunState::load(&id).unwrap().passes.is_empty());
18474        drop(lease);
18475        let mut execution = resumed.claim_driver().await.unwrap();
18476        execution
18477            .driver
18478            .finish(&mut execution.runner.state)
18479            .await
18480            .unwrap();
18481    }
18482
18483    #[tokio::test]
18484    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18485        crate::run::pin_test_home();
18486        let mut runner = runner_at(RunStatus::Reviewing);
18487        let state = &mut runner.state;
18488        state.config.agents = vec![conductor()];
18489        state.driver_claimed = None;
18490        state.driver_pid = Some(std::process::id());
18491        state.driver_started_at = None;
18492        state.begin_pass();
18493        state.save().unwrap();
18494        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18495        std::fs::write(
18496            state.dir().join("driver.json"),
18497            serde_json::to_vec(&serde_json::json!({
18498                "pid": state.driver_pid, "started_at": null,
18499                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18500            }))
18501            .unwrap(),
18502        )
18503        .unwrap();
18504        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18505        drop(lease);
18506        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18507        let mut resumed = Runner::resume(&state.id).unwrap();
18508        assert!(!resumed.state.driver_exited);
18509        let mut execution = resumed
18510            .claim_driver()
18511            .await
18512            .expect("lease proves previous execution ended");
18513        assert_eq!(execution.runner.state.passes.len(), 2);
18514        execution
18515            .driver
18516            .finish(&mut execution.runner.state)
18517            .await
18518            .unwrap();
18519    }
18520
18521    #[tokio::test]
18522    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18523        crate::run::pin_test_home();
18524        let mut runner = runner_at(RunStatus::Reviewing);
18525        let mut json = serde_json::to_value(&runner.state).unwrap();
18526        json.as_object_mut().unwrap().remove("driver_claimed");
18527        runner.state = serde_json::from_value(json).unwrap();
18528        assert_eq!(runner.state.driver_claimed, None);
18529        runner.state.save().unwrap();
18530        let mut execution = runner
18531            .claim_driver()
18532            .await
18533            .expect("the execution lease, not a missing marker, excludes a second driver");
18534        assert_eq!(execution.runner.state.passes.len(), 1);
18535        execution
18536            .driver
18537            .finish(&mut execution.runner.state)
18538            .await
18539            .unwrap();
18540    }
18541
18542    #[tokio::test]
18543    async fn driver_admission_refuses_a_proven_live_driver() {
18544        crate::run::pin_test_home();
18545        let mut runner = runner_at(RunStatus::Reviewing);
18546        let pid = std::process::id();
18547        runner.state.driver_pid = Some(pid);
18548        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18549        assert!(runner.state.driver_started_at.is_some());
18550        runner.state.save().unwrap();
18551        let error = runner
18552            .claim_driver()
18553            .await
18554            .err()
18555            .expect("a live driver must refuse");
18556        assert!(error.to_string().contains("live driver"), "{error:#}");
18557        let saved = RunState::load(&runner.state.id).unwrap();
18558        assert_eq!(saved.driver_pid, Some(pid));
18559        assert_eq!(saved.status, RunStatus::Reviewing);
18560        assert!(saved.passes.is_empty());
18561    }
18562
18563    #[test]
18564    fn parking_resource_halt_persists_the_pass_once() {
18565        crate::run::pin_test_home();
18566        let mut runner = runner_at(RunStatus::Reviewing);
18567        runner.state.begin_pass();
18568        runner.state.resource_block = Some(crate::run::ResourceBlock {
18569            node: "fix".into(),
18570            kind: "shared build cache".into(),
18571            detail: "cancelled".into(),
18572            at: Timestamp::now(),
18573            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18574        });
18575        let cause = ParkCause::Upgrade {
18576            from: "old".into(),
18577            to: Some("new".into()),
18578        };
18579        runner.pause.park_for(cause.clone());
18580        // Both direct fixer exits and graph resource boundaries must complete
18581        // parking, rather than returning a stale Reviewing record.
18582        runner.stop_for_resource().unwrap();
18583        assert!(runner.resource_halt().unwrap());
18584        assert!(runner.park_here().unwrap());
18585        let saved = RunState::load(&runner.state.id).unwrap();
18586        assert!(saved.parked);
18587        assert_eq!(saved.status, RunStatus::Reviewing);
18588        assert!(saved.passes[0].ended_at.is_some());
18589        assert_eq!(
18590            saved.passes[0].end,
18591            Some(crate::run::PassEnd::Parked {
18592                cause: cause.clone()
18593            })
18594        );
18595        assert_eq!(
18596            saved.resource_block.unwrap().cause,
18597            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18598        );
18599        assert_eq!(
18600            saved
18601                .events
18602                .iter()
18603                .filter(|event| event.node == "park")
18604                .count(),
18605            1
18606        );
18607    }
18608
18609    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
18610    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
18611    /// cleared) must not make a *different* run - one only watching its own,
18612    /// unshared interrupt `Pause` - see itself as parked. If a future change
18613    /// ever collapsed these back into one handle, the interrupt scheduler
18614    /// would park every run for the rest of the daemon's life, not just the
18615    /// one it meant to interrupt.
18616    #[test]
18617    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18618        crate::run::pin_test_home();
18619        let mut runner = runner_at(RunStatus::Implementing);
18620        let shutdown = Pause::new();
18621        runner.on_pause(shutdown.clone());
18622        let interrupt = Pause::new();
18623        runner.watch_interrupt(interrupt.clone());
18624
18625        // Nobody has asked for anything yet.
18626        assert!(!runner.park_here().expect("park_here"));
18627        assert!(!runner.state.parked);
18628
18629        // Only the interrupt handle fires; the shutdown handle stays clear.
18630        interrupt.park_because("test");
18631        assert!(!shutdown.parked());
18632        assert!(runner.park_here().expect("park_here"));
18633    }
18634
18635    /// The property every prior attempt at this feature failed to pin down:
18636    /// asking a run to park while one of its nodes has a real, in-flight
18637    /// async operation running (an agent call, in production) must not cut
18638    /// that operation short. `park_here` is only ever consulted *between*
18639    /// `execute`'s node calls - see its own doc - so nothing inside a node
18640    /// can observe a park request until the node itself returns. This proves
18641    /// that structurally, with real `tokio` concurrency and a channel
18642    /// handshake (never a sleep, which would only prove "usually", not
18643    /// "cannot"): the "node" below reports that it has genuinely started,
18644    /// and only then is the park requested; the node still has to be told to
18645    /// finish before `park_here` is ever called, exactly mirroring every
18646    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
18647    /// in `execute`.
18648    #[tokio::test]
18649    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18650        crate::run::pin_test_home();
18651        let mut runner = runner_at(RunStatus::Implementing);
18652        let interrupt = Pause::new();
18653        runner.watch_interrupt(interrupt.clone());
18654
18655        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18656        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18657
18658        // Stands in for one node's in-flight agent call: it proves it has
18659        // genuinely started, then blocks - exactly as a spawned CLI process
18660        // does - until told to finish.
18661        let node = async move {
18662            started_tx.send(()).expect("send started");
18663            finish_rx.await.expect("recv finish");
18664            "node finished"
18665        };
18666
18667        let interrupter = async move {
18668            started_rx.await.expect("recv started");
18669            // The call is now genuinely in flight. Ask it to park.
18670            interrupt.park_because("higher-priority task waiting");
18671            // Nothing the node does can observe this yet - there is no
18672            // check inside it, by construction - so let the executor run
18673            // anything pending and then let the node finish on its own.
18674            tokio::task::yield_now().await;
18675            finish_tx.send(()).expect("send finish");
18676        };
18677
18678        let (node_result, ()) = tokio::join!(node, interrupter);
18679        assert_eq!(
18680            node_result, "node finished",
18681            "the in-flight call ran to completion"
18682        );
18683
18684        // Only now, at the boundary the real `execute` would check right
18685        // after this node, does the park take effect.
18686        assert!(runner.park_here().expect("park_here"));
18687        assert!(runner.state.parked);
18688    }
18689
18690    /// A run parked mid-competition carries every field it had accumulated
18691    /// through the exact same disk round-trip an ordinary resume uses -
18692    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
18693    /// Nothing about parking for an interrupt is a special case of that path;
18694    /// this is what proves it rather than assuming it.
18695    #[test]
18696    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18697        crate::run::pin_test_home();
18698        let mut runner = runner_at(RunStatus::Judging);
18699        // `Runner::resume` re-resolves roles from the saved config, which
18700        // refuses an empty roster - give it the same minimal one `conductor`
18701        // itself uses.
18702        runner.state.config.agents = vec![conductor()];
18703        runner.state.candidates = vec![Candidate {
18704            index: 0,
18705            label: 'A',
18706            agent: "alpha".to_owned(),
18707            branch: "magi/x/A".to_owned(),
18708            worktree: PathBuf::from("/nonexistent/worktree"),
18709            summary: "did the thing".to_owned(),
18710            stat: "1 file changed".to_owned(),
18711            files: 1,
18712            commits: 1,
18713            empty: false,
18714            failed: None,
18715            verified_noop: None,
18716            duration_ms: 1234,
18717            folded: false,
18718        }];
18719        let run_id = runner.state.id.clone();
18720
18721        let interrupt = Pause::new();
18722        runner.watch_interrupt(interrupt.clone());
18723        interrupt.park_because("task c3d4 asked to run first");
18724        assert!(runner.park_here().expect("park_here"));
18725
18726        let resumed = Runner::resume(&run_id).expect("resume");
18727        assert_eq!(resumed.state.candidates.len(), 1);
18728        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18729        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18730        assert_eq!(resumed.state.status, runner.state.status);
18731        assert!(
18732            resumed.state.parked,
18733            "still parked until `execute` actually walks the graph again"
18734        );
18735        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18736    }
18737
18738    /// A fresh open question on `run`, stored and handed back for assertions.
18739    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18740        let mut q = ask::Question::new(
18741            run.to_owned(),
18742            "implement".to_owned(),
18743            "impl-A".to_owned(),
18744            "Which storage backend should the cache use?".to_owned(),
18745            String::new(),
18746            vec!["SQLite".to_owned(), "Redis".to_owned()],
18747        );
18748        store.put(&mut q).unwrap();
18749        q
18750    }
18751
18752    #[test]
18753    fn a_failed_runs_open_question_is_abandoned() {
18754        ask_test_home();
18755        let store = ask::Questions::open();
18756        let mut runner = runner_at(RunStatus::Failed);
18757        let run = runner.state.id.clone();
18758        let q = ask_open_question(&store, &run);
18759
18760        runner.settle_questions();
18761
18762        let back = store.get(&q.id).unwrap();
18763        assert!(
18764            !back.status.open(),
18765            "the seat that asked died with the run; nobody is left to read an answer"
18766        );
18767        assert!(
18768            back.detail.contains(&run) && back.detail.contains("failed"),
18769            "the reason names what the run became, not just that it is gone: {}",
18770            back.detail
18771        );
18772    }
18773
18774    #[test]
18775    fn a_merged_runs_open_question_is_abandoned_too() {
18776        ask_test_home();
18777        let store = ask::Questions::open();
18778        // A run that finishes cleanly still leaves nobody to read an answer -
18779        // this is not only a failure-path cleanup.
18780        for status in [RunStatus::Merged, RunStatus::Ready] {
18781            let mut runner = runner_at(status);
18782            let run = runner.state.id.clone();
18783            let q = ask_open_question(&store, &run);
18784
18785            runner.settle_questions();
18786
18787            let back = store.get(&q.id).unwrap();
18788            assert!(
18789                !back.status.open(),
18790                "{status:?} run's question must not outlive the run"
18791            );
18792        }
18793    }
18794
18795    #[test]
18796    fn a_still_resumable_runs_open_question_is_left_alone() {
18797        ask_test_home();
18798        let store = ask::Questions::open();
18799        // `Blocked` and `Stalled` can still be resumed — the candidates, the
18800        // review round and the seat sessions are all still on disk — so a
18801        // question asked mid-round may yet get a real answer from a real
18802        // resume. Sweeping it here would be exactly the failure mode this
18803        // whole feature exists to avoid on the other side.
18804        for status in [RunStatus::Blocked, RunStatus::Stalled] {
18805            let mut runner = runner_at(status);
18806            let run = runner.state.id.clone();
18807            let q = ask_open_question(&store, &run);
18808
18809            runner.settle_questions();
18810
18811            let back = store.get(&q.id).unwrap();
18812            assert!(
18813                back.status.open(),
18814                "{status:?} is still alive; the question must still be waiting"
18815            );
18816        }
18817    }
18818
18819    #[test]
18820    fn settle_questions_never_touches_an_already_answered_question() {
18821        ask_test_home();
18822        let store = ask::Questions::open();
18823        let mut runner = runner_at(RunStatus::Failed);
18824        let run = runner.state.id.clone();
18825        let mut q = ask_open_question(&store, &run);
18826        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
18827            .unwrap();
18828        store.put(&mut q).unwrap();
18829
18830        // Called twice, the way a crash-recovered daemon reclaim and the
18831        // graph's own cleanup both can for the same run — `abandon_for_run`
18832        // only ever touches what is still open, so this must be inert both
18833        // times, not merely the second.
18834        runner.settle_questions();
18835        runner.settle_questions();
18836
18837        let back = store.get(&q.id).unwrap();
18838        assert_eq!(
18839            back.status,
18840            ask::QuestionStatus::Answered,
18841            "a real answer is a decision on record, never overwritten by a sweep"
18842        );
18843    }
18844
18845    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
18846    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
18847    /// without merging, whose winner is still the operator's answer to read.
18848    /// Nothing previously called `fold_run` itself with a real `tally`, so
18849    /// this is the first test to pin down the one distinction the whole
18850    /// automatic-fold feature depends on: the winner's worktree and branch
18851    /// must survive, everything else sharing the run's worktree bay - a
18852    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
18853    /// second sweep treats every non-winner directory under the bay alike -
18854    /// must not.
18855    #[tokio::test]
18856    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
18857        crate::run::pin_test_home();
18858        let tmp = tempfile::tempdir().expect("tempdir");
18859        let repo = tmp.path().join("repo");
18860        std::fs::create_dir_all(&repo).unwrap();
18861        init_repo(&repo);
18862
18863        let mut config = Config::default();
18864        config.graph.worktree_root = Some(tmp.path().join("wt"));
18865
18866        let mut state = RunState::new(
18867            repo.clone(),
18868            "main".to_owned(),
18869            "deadbeef".to_owned(),
18870            "task".to_owned(),
18871            config,
18872        );
18873        let root = state.worktree_root();
18874        let wt_a = root.join("cand-A");
18875        let wt_b = root.join("cand-B");
18876        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
18877            .await
18878            .expect("worktree A");
18879        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
18880            .await
18881            .expect("worktree B");
18882
18883        state.candidates = vec![
18884            Candidate {
18885                index: 0,
18886                label: 'A',
18887                agent: "alpha".to_owned(),
18888                branch: "magi/x/A".to_owned(),
18889                worktree: wt_a.clone(),
18890                summary: String::new(),
18891                stat: String::new(),
18892                files: 0,
18893                commits: 0,
18894                empty: false,
18895                failed: None,
18896                verified_noop: None,
18897                duration_ms: 0,
18898                folded: false,
18899            },
18900            Candidate {
18901                index: 1,
18902                label: 'B',
18903                agent: "beta".to_owned(),
18904                branch: "magi/x/B".to_owned(),
18905                worktree: wt_b.clone(),
18906                summary: String::new(),
18907                stat: String::new(),
18908                files: 0,
18909                commits: 0,
18910                empty: false,
18911                failed: None,
18912                verified_noop: None,
18913                duration_ms: 0,
18914                folded: false,
18915            },
18916        ];
18917        state.tally = Some(Tally {
18918            first_choice: BTreeMap::from([('A', 1)]),
18919            borda: BTreeMap::new(),
18920            winner: 'A',
18921            rankings: 1,
18922            unanimous_initial: true,
18923            deliberated: false,
18924            changed_votes: 0,
18925            unanimous_final: true,
18926            tie_break: None,
18927            judges: 1,
18928            present: 1,
18929            quorum: 1,
18930            met_quorum: true,
18931            uncontested: None,
18932        });
18933        state.status = RunStatus::Ready;
18934
18935        fold_run(&mut state, false, &crate::run::home())
18936            .await
18937            .expect("fold_run");
18938
18939        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
18940        assert!(
18941            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
18942            "the unmerged winner's branch survives"
18943        );
18944        assert!(
18945            !state.candidates[0].folded,
18946            "the winner is not marked folded"
18947        );
18948
18949        assert!(!wt_b.exists(), "the loser's worktree is removed");
18950        assert!(
18951            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
18952            "the loser's branch is removed"
18953        );
18954        assert!(state.candidates[1].folded, "the loser is marked folded");
18955    }
18956
18957    #[tokio::test]
18958    #[cfg(unix)]
18959    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
18960        use std::os::fd::AsRawFd as _;
18961        use std::time::Duration;
18962
18963        let tmp = tempfile::tempdir().unwrap();
18964        let repo = tmp.path().join("repo");
18965        std::fs::create_dir_all(&repo).unwrap();
18966        init_repo(&repo);
18967        let common = repo.join(".git");
18968        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
18969        let home = tmp.path().join("home");
18970        let mut config = Config::default();
18971        config.coordination = crate::config::test_coordination();
18972        config.graph.worktree_root = Some(tmp.path().join("wt"));
18973        config.graph.timeout_resource_wait = 1;
18974        let mut state = RunState::new(
18975            repo.clone(),
18976            "main".into(),
18977            "deadbeef".into(),
18978            "task".into(),
18979            config,
18980        );
18981        let root = state.worktree_root();
18982        let candidate = root.join("cand-A");
18983        let reviewer = root.join("review-1");
18984        let branch = state.branch_for('A');
18985        git::worktree_add_branch(&repo, &candidate, &branch, "main")
18986            .await
18987            .unwrap();
18988        git::worktree_add_detached(&repo, &reviewer, "main")
18989            .await
18990            .unwrap();
18991        git::acquire_worktree_config(&repo).await.unwrap();
18992        state.enabled_worktree_config = true;
18993        state.status = RunStatus::Ready;
18994        state.candidates.push(Candidate {
18995            index: 0,
18996            label: 'A',
18997            agent: "implement".into(),
18998            branch: branch.clone(),
18999            worktree: candidate.clone(),
19000            summary: String::new(),
19001            stat: String::new(),
19002            files: 0,
19003            commits: 0,
19004            empty: false,
19005            failed: None,
19006            verified_noop: None,
19007            duration_ms: 0,
19008            folded: false,
19009        });
19010        state.save_under(&home).unwrap();
19011        let config_before = std::fs::read(common.join("config")).unwrap();
19012        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19013            .await
19014            .unwrap();
19015        // A separate OS handle bypasses the launcher's local gate, exactly as
19016        // an external updater does, and must still prevent all folding work.
19017        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19018        assert_eq!(
19019            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19020            0
19021        );
19022        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19023        assert!(
19024            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19025                .await
19026                .is_err()
19027        );
19028        assert!(candidate.exists());
19029        assert!(reviewer.exists());
19030        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19031        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19032        assert_eq!(
19033            git::git(&repo, &["worktree", "list", "--porcelain"])
19034                .await
19035                .unwrap(),
19036            worktrees_before
19037        );
19038        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19039            .await
19040            .unwrap()
19041            .unwrap_err();
19042        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19043        drop(fold);
19044        assert!(candidate.exists() && reviewer.exists());
19045        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19046        assert_eq!(
19047            state.resource_block.as_ref().unwrap().kind,
19048            "repository coordination"
19049        );
19050        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19051        drop(external);
19052        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19053        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19054            .await
19055            .unwrap()
19056            .unwrap();
19057        drop(fold);
19058        assert!(removed.contains(&branch));
19059        assert!(!root.exists());
19060        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19061        assert!(state.candidates[0].folded);
19062        assert!(!state.enabled_worktree_config);
19063        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19064        let enabled = git::git_raw(
19065            &repo,
19066            &["config", "--local", "--get", "extensions.worktreeConfig"],
19067        )
19068        .await
19069        .unwrap();
19070        assert!(!enabled.ok(), "the last config reference was released");
19071    }
19072
19073    /// A branch handed to a later run is that run's (and its pull request's):
19074    /// folding the run that released it must not delete it.
19075    #[tokio::test]
19076    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19077        let tmp = tempfile::tempdir().expect("tempdir");
19078        let repo = tmp.path().join("repo");
19079        std::fs::create_dir_all(&repo).unwrap();
19080        init_repo(&repo);
19081        let home = tmp.path().join("home");
19082
19083        let mut config = Config::default();
19084        config.graph.worktree_root = Some(tmp.path().join("wt"));
19085        let mut state = RunState::new(
19086            repo.clone(),
19087            "main".to_owned(),
19088            "deadbeef".to_owned(),
19089            "task".to_owned(),
19090            config,
19091        );
19092        // The worktree is already gone (released); the branch survives.
19093        git::git(&repo, &["branch", "magi/x/A", "main"])
19094            .await
19095            .expect("branch");
19096        state.candidates = vec![Candidate {
19097            index: 0,
19098            label: 'A',
19099            agent: "alpha".to_owned(),
19100            branch: "magi/x/A".to_owned(),
19101            worktree: state.worktree_root().join("cand-A"),
19102            summary: String::new(),
19103            stat: String::new(),
19104            files: 0,
19105            commits: 0,
19106            empty: false,
19107            failed: None,
19108            verified_noop: None,
19109            duration_ms: 0,
19110            folded: true,
19111        }];
19112        state.released_to = Some("20260901-000000-new1".to_owned());
19113        state.released_branches = vec!["magi/x/A".to_owned()];
19114
19115        fold_run(&mut state, true, &home).await.expect("fold_run");
19116        state.status = RunStatus::Merged;
19117        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19118        state.merge = Some(crate::run::MergeOutcome {
19119            mode: MergeMode::Pr,
19120            ok: true,
19121            detail: "confirmed merged".into(),
19122            empty: false,
19123        });
19124        fold_run(&mut state, true, &home)
19125            .await
19126            .expect("merged fold");
19127
19128        assert!(
19129            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19130            "the handed-over branch survives a fold"
19131        );
19132    }
19133
19134    /// A winner with nothing ahead of the base is caught before `gh` is ever
19135    /// asked for a pull request, and the message carries what the task's
19136    /// references resolved to.
19137    #[tokio::test]
19138    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19139        let tmp = tempfile::tempdir().expect("tempdir");
19140        let repo = tmp.path().join("repo");
19141        std::fs::create_dir_all(&repo).unwrap();
19142        init_repo(&repo);
19143        let run = |args: &[&str]| {
19144            let out = std::process::Command::new("git")
19145                .quiet()
19146                .args(args)
19147                .current_dir(&repo)
19148                .output()
19149                .expect("spawn git");
19150            assert!(out.status.success(), "git {args:?}");
19151        };
19152        run(&["branch", "magi/x/A"]);
19153        run(&["checkout", "-q", "-b", "magi/x/B"]);
19154        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19155        run(&["add", "-A"]);
19156        run(&["commit", "-q", "-m", "work"]);
19157        run(&["checkout", "-q", "main"]);
19158
19159        // A pull request is compared against the remote's base, so the
19160        // fixture needs one. Before it exists nothing can be read, and the
19161        // answer must be "not empty".
19162        let probe = RunState::new(
19163            repo.clone(),
19164            "main".to_owned(),
19165            "deadbeef".to_owned(),
19166            "task".to_owned(),
19167            Config::default(),
19168        );
19169        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19170        let bare = tmp.path().join("origin.git");
19171        let out = std::process::Command::new("git")
19172            .quiet()
19173            .args(["init", "-q", "--bare"])
19174            .arg(&bare)
19175            .output()
19176            .expect("spawn git");
19177        assert!(out.status.success(), "git init --bare");
19178        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19179        run(&["push", "-q", "origin", "main"]);
19180
19181        let mut state = RunState::new(
19182            repo.clone(),
19183            "main".to_owned(),
19184            "deadbeef".to_owned(),
19185            "task".to_owned(),
19186            Config::default(),
19187        );
19188        state.seeds = vec![refs::Seed {
19189            token: "magi/27b2/A".to_owned(),
19190            kind: refs::SeedKind::Unresolved,
19191            sha: String::new(),
19192            branch: true,
19193            detail: "no branch or commit named magi/27b2/A".to_owned(),
19194        }];
19195
19196        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19197        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19198        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19199        let detail = empty_candidate_detail(&state, "main");
19200        assert!(detail.starts_with("empty candidate"), "{detail}");
19201        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19202        assert!(detail.contains("--start-from"), "{detail}");
19203    }
19204
19205    /// `status == Ready` used to be read as "this is the harmless
19206    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19207    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19208    /// run's PR was closed without merging — and reentering `merge` with
19209    /// `mode` still `Pr` does not know the difference, so it pushed and
19210    /// opened a second pull request. `mode == Local` reproduces the same
19211    /// blind spot without a network call: reentry must not attempt another
19212    /// git merge once this node has already recorded an outcome.
19213    #[tokio::test]
19214    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19215        let tmp = tempfile::tempdir().expect("tempdir");
19216        let repo = tmp.path().join("repo");
19217        std::fs::create_dir_all(&repo).unwrap();
19218        init_repo(&repo);
19219
19220        let mut config = Config::default();
19221        config.merge.mode = MergeMode::Local;
19222
19223        let mut state = RunState::new(
19224            repo.clone(),
19225            "main".to_owned(),
19226            "deadbeef".to_owned(),
19227            "task".to_owned(),
19228            config,
19229        );
19230        state.candidates = vec![Candidate {
19231            index: 0,
19232            label: 'A',
19233            agent: "alpha".to_owned(),
19234            branch: "does-not-exist".to_owned(),
19235            worktree: repo.clone(),
19236            summary: String::new(),
19237            stat: String::new(),
19238            files: 0,
19239            commits: 0,
19240            empty: false,
19241            failed: None,
19242            verified_noop: None,
19243            duration_ms: 0,
19244            folded: false,
19245        }];
19246        state.tally = Some(Tally {
19247            first_choice: BTreeMap::from([('A', 1)]),
19248            borda: BTreeMap::new(),
19249            winner: 'A',
19250            rankings: 1,
19251            unanimous_initial: true,
19252            deliberated: false,
19253            changed_votes: 0,
19254            unanimous_final: true,
19255            tie_break: None,
19256            judges: 0,
19257            present: 0,
19258            quorum: 0,
19259            met_quorum: true,
19260            uncontested: Some("only candidate A produced a change".to_owned()),
19261        });
19262        state.reviews = vec![ReviewRound {
19263            round: 1,
19264            head: "deadbeef".to_owned(),
19265            verified_head: None,
19266            verified_at: None,
19267            reviews: Vec::new(),
19268            e2e: Vec::new(),
19269            fix: None,
19270            blocking: 0,
19271            answered: 0,
19272            expected: 0,
19273            clean: true,
19274            verify_retried: false,
19275            e2e_deferred: false,
19276            e2e_defer_reason: None,
19277            progressed: false,
19278            vote_split: false,
19279            reconsideration: Vec::new(),
19280            verdict: None,
19281        }];
19282        state.gate = vec![CommandOutcome {
19283            command: "test".to_owned(),
19284            code: Some(0),
19285            output_tail: String::new(),
19286            duration_ms: 0,
19287            resource_blocked: false,
19288        }];
19289        state.gate_ran = true;
19290        // Reached its conclusion already — e.g. `land` closing the PR without
19291        // merging it, which (like the honest `MergeMode::None` path) leaves
19292        // `status` at `Ready`. The recorded outcome is what actually marks
19293        // this node done.
19294        state.status = RunStatus::Ready;
19295        state.merge = Some(MergeOutcome {
19296            mode: MergeMode::Local,
19297            ok: false,
19298            detail: "already concluded".to_owned(),
19299            empty: false,
19300        });
19301
19302        let mut runner = Runner {
19303            state,
19304            roles: ResolvedRoles {
19305                implementers: Vec::new(),
19306                judges: Vec::new(),
19307                reviewers: Vec::new(),
19308                fixer: None,
19309                conductor: conductor(),
19310                implementer_roster: Vec::new(),
19311                judge_roster: Vec::new(),
19312                reviewer_roster: Vec::new(),
19313            },
19314            sem: Arc::new(Semaphore::new(1)),
19315            pause: Pause::new(),
19316            interrupt: Pause::new(),
19317        };
19318
19319        runner.merge().await.expect("merge");
19320
19321        assert_eq!(
19322            runner.state.status,
19323            RunStatus::Ready,
19324            "a concluded run's status must not change on reentry"
19325        );
19326        assert_eq!(
19327            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19328            Some("already concluded"),
19329            "merge must not run again once the node already recorded an outcome"
19330        );
19331    }
19332
19333    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19334    /// when its last attempt was resource-blocked (the shared build cache
19335    /// could not be acquired or confirmed fresh in time - see
19336    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19337    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19338    /// a run merge a tree the gate never actually checked - exactly the case
19339    /// a contended cache produces on every retry until it clears. `merge`
19340    /// must refuse until `gate` has actually recorded an attempt.
19341    #[tokio::test]
19342    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19343        let tmp = tempfile::tempdir().expect("tempdir");
19344        let repo = tmp.path().join("repo");
19345        std::fs::create_dir_all(&repo).unwrap();
19346        init_repo(&repo);
19347
19348        let mut config = Config::default();
19349        config.merge.mode = MergeMode::Local;
19350
19351        let mut state = RunState::new(
19352            repo.clone(),
19353            "main".to_owned(),
19354            "deadbeef".to_owned(),
19355            "task".to_owned(),
19356            config,
19357        );
19358        state.candidates = vec![Candidate {
19359            index: 0,
19360            label: 'A',
19361            agent: "alpha".to_owned(),
19362            branch: "does-not-exist".to_owned(),
19363            worktree: repo.clone(),
19364            summary: String::new(),
19365            stat: String::new(),
19366            files: 0,
19367            commits: 0,
19368            empty: false,
19369            failed: None,
19370            verified_noop: None,
19371            duration_ms: 0,
19372            folded: false,
19373        }];
19374        state.tally = Some(Tally {
19375            first_choice: BTreeMap::from([('A', 1)]),
19376            borda: BTreeMap::new(),
19377            winner: 'A',
19378            rankings: 1,
19379            unanimous_initial: true,
19380            deliberated: false,
19381            changed_votes: 0,
19382            unanimous_final: true,
19383            tie_break: None,
19384            judges: 0,
19385            present: 0,
19386            quorum: 0,
19387            met_quorum: true,
19388            uncontested: Some("only candidate A produced a change".to_owned()),
19389        });
19390        state.reviews = vec![ReviewRound {
19391            round: 1,
19392            head: "deadbeef".to_owned(),
19393            verified_head: None,
19394            verified_at: None,
19395            reviews: Vec::new(),
19396            e2e: Vec::new(),
19397            fix: None,
19398            blocking: 0,
19399            answered: 0,
19400            expected: 0,
19401            clean: true,
19402            verify_retried: false,
19403            e2e_deferred: false,
19404            e2e_defer_reason: None,
19405            progressed: false,
19406            vote_split: false,
19407            reconsideration: Vec::new(),
19408            verdict: None,
19409        }];
19410        // The point: `gate` has not recorded anything yet.
19411        state.gate = Vec::new();
19412        state.gate_ran = false;
19413        state.status = RunStatus::Gating;
19414
19415        let mut runner = Runner {
19416            state,
19417            roles: ResolvedRoles {
19418                implementers: Vec::new(),
19419                judges: Vec::new(),
19420                reviewers: Vec::new(),
19421                fixer: None,
19422                conductor: conductor(),
19423                implementer_roster: Vec::new(),
19424                judge_roster: Vec::new(),
19425                reviewer_roster: Vec::new(),
19426            },
19427            sem: Arc::new(Semaphore::new(1)),
19428            pause: Pause::new(),
19429            interrupt: Pause::new(),
19430        };
19431
19432        runner.merge().await.expect("merge");
19433
19434        assert!(
19435            runner.state.merge.is_none(),
19436            "an empty gate must never be read as a passing one: {:?}",
19437            runner.state.merge
19438        );
19439    }
19440
19441    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19442    /// commands configured and `merge.mode` is `none` (a review-only run).
19443    /// `gate` must still record a real attempt — zero commands, vacuously
19444    /// passed — rather than leaving `state.gate` empty in a way `merge`
19445    /// cannot tell apart from "never ran"; otherwise the run reaches
19446    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19447    #[tokio::test]
19448    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19449        let tmp = tempfile::tempdir().expect("tempdir");
19450        let repo = tmp.path().join("repo");
19451        std::fs::create_dir_all(&repo).unwrap();
19452        init_repo(&repo);
19453
19454        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19455        let config = Config::default();
19456
19457        let mut state = RunState::new(
19458            repo.clone(),
19459            "main".to_owned(),
19460            "deadbeef".to_owned(),
19461            "task".to_owned(),
19462            config,
19463        );
19464        state.candidates = vec![Candidate {
19465            index: 0,
19466            label: 'A',
19467            agent: "alpha".to_owned(),
19468            branch: "does-not-exist".to_owned(),
19469            worktree: repo.clone(),
19470            summary: String::new(),
19471            stat: String::new(),
19472            files: 0,
19473            commits: 0,
19474            empty: false,
19475            failed: None,
19476            verified_noop: None,
19477            duration_ms: 0,
19478            folded: false,
19479        }];
19480        state.tally = Some(Tally {
19481            first_choice: BTreeMap::from([('A', 1)]),
19482            borda: BTreeMap::new(),
19483            winner: 'A',
19484            rankings: 1,
19485            unanimous_initial: true,
19486            deliberated: false,
19487            changed_votes: 0,
19488            unanimous_final: true,
19489            tie_break: None,
19490            judges: 0,
19491            present: 0,
19492            quorum: 0,
19493            met_quorum: true,
19494            uncontested: Some("only candidate A produced a change".to_owned()),
19495        });
19496        state.reviews = vec![ReviewRound {
19497            round: 1,
19498            head: "deadbeef".to_owned(),
19499            verified_head: None,
19500            verified_at: None,
19501            reviews: Vec::new(),
19502            e2e: Vec::new(),
19503            fix: None,
19504            blocking: 0,
19505            answered: 0,
19506            expected: 0,
19507            clean: true,
19508            verify_retried: false,
19509            e2e_deferred: false,
19510            e2e_defer_reason: None,
19511            progressed: false,
19512            vote_split: false,
19513            reconsideration: Vec::new(),
19514            verdict: None,
19515        }];
19516
19517        let mut runner = Runner {
19518            state,
19519            roles: ResolvedRoles {
19520                implementers: Vec::new(),
19521                judges: Vec::new(),
19522                reviewers: Vec::new(),
19523                fixer: None,
19524                conductor: conductor(),
19525                implementer_roster: Vec::new(),
19526                judge_roster: Vec::new(),
19527                reviewer_roster: Vec::new(),
19528            },
19529            sem: Arc::new(Semaphore::new(1)),
19530            pause: Pause::new(),
19531            interrupt: Pause::new(),
19532        };
19533
19534        runner.gate().await.expect("gate");
19535        assert!(
19536            runner.state.gate_ran,
19537            "zero configured commands is still a real attempt, not an unrun gate"
19538        );
19539        assert!(runner.state.gate.is_empty());
19540        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19541        assert_ne!(
19542            runner.state.status,
19543            RunStatus::Blocked,
19544            "a gate with nothing to check must not read as failed"
19545        );
19546
19547        runner.merge().await.expect("merge");
19548        assert_eq!(
19549            runner.state.status,
19550            RunStatus::Ready,
19551            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19552        );
19553    }
19554
19555    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19556    /// `verify.gate` (so the e2e leg and the final gate never build against
19557    /// different directories). With zero `verify.gate` commands but a
19558    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19559    /// that lease before discovering it had nothing to run - so a repo with
19560    /// no gate commands could come back `resource_blocked` (and therefore
19561    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19562    /// cache, exactly the contention this run's own zero commands could
19563    /// never have touched. `gate` must recognise there is nothing to check
19564    /// before it ever asks for the lease.
19565    #[tokio::test]
19566    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19567        crate::run::pin_test_home();
19568        let home = crate::run::home();
19569
19570        let tmp = tempfile::tempdir().expect("tempdir");
19571        let repo = tmp.path().join("repo");
19572        std::fs::create_dir_all(&repo).unwrap();
19573        init_repo(&repo);
19574        // Unique to this test, so holding its lease cannot collide with
19575        // another test sharing the same process-wide `home`.
19576        let cache_dir = tmp.path().join("target");
19577
19578        let mut config = Config::default();
19579        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19580        // `verify.gate` stays empty (the default). Any accidental cache
19581        // acquisition must report contention immediately, regardless of how
19582        // long unrelated Git subprocesses take under parallel test load.
19583        config.graph.timeout_verify = Some(2);
19584        config.graph.timeout_resource_wait = 0;
19585
19586        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19587        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19588            .expect("no io error acquiring directly")
19589        {
19590            crate::cache::AcquireOutcome::Acquired(g) => g,
19591            crate::cache::AcquireOutcome::Busy(b) => {
19592                panic!("expected the direct acquire to win the lease first: {b:?}")
19593            }
19594        };
19595
19596        let mut state = RunState::new(
19597            repo.clone(),
19598            "main".to_owned(),
19599            "deadbeef".to_owned(),
19600            "task".to_owned(),
19601            config,
19602        );
19603        state.candidates = vec![Candidate {
19604            index: 0,
19605            label: 'A',
19606            agent: "alpha".to_owned(),
19607            branch: "does-not-exist".to_owned(),
19608            worktree: repo.clone(),
19609            summary: String::new(),
19610            stat: String::new(),
19611            files: 0,
19612            commits: 0,
19613            empty: false,
19614            failed: None,
19615            verified_noop: None,
19616            duration_ms: 0,
19617            folded: false,
19618        }];
19619        state.tally = Some(Tally {
19620            first_choice: BTreeMap::from([('A', 1)]),
19621            borda: BTreeMap::new(),
19622            winner: 'A',
19623            rankings: 1,
19624            unanimous_initial: true,
19625            deliberated: false,
19626            changed_votes: 0,
19627            unanimous_final: true,
19628            tie_break: None,
19629            judges: 0,
19630            present: 0,
19631            quorum: 0,
19632            met_quorum: true,
19633            uncontested: Some("only candidate A produced a change".to_owned()),
19634        });
19635        state.reviews = vec![ReviewRound {
19636            round: 1,
19637            head: "deadbeef".to_owned(),
19638            verified_head: None,
19639            verified_at: None,
19640            reviews: Vec::new(),
19641            e2e: Vec::new(),
19642            fix: None,
19643            blocking: 0,
19644            answered: 0,
19645            expected: 0,
19646            clean: true,
19647            verify_retried: false,
19648            e2e_deferred: false,
19649            e2e_defer_reason: None,
19650            progressed: false,
19651            vote_split: false,
19652            reconsideration: Vec::new(),
19653            verdict: None,
19654        }];
19655
19656        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19657        state.reviews[0].e2e = vec![CommandOutcome {
19658            command: state.config.verify.e2e[0].clone(),
19659            code: Some(0),
19660            output_tail: String::new(),
19661            duration_ms: 0,
19662            resource_blocked: false,
19663        }];
19664        let mut runner = Runner {
19665            state,
19666            roles: ResolvedRoles {
19667                implementers: Vec::new(),
19668                judges: Vec::new(),
19669                reviewers: Vec::new(),
19670                fixer: None,
19671                conductor: conductor(),
19672                implementer_roster: Vec::new(),
19673                judge_roster: Vec::new(),
19674                reviewer_roster: Vec::new(),
19675            },
19676            sem: Arc::new(Semaphore::new(1)),
19677            pause: Pause::new(),
19678            interrupt: Pause::new(),
19679        };
19680
19681        runner.gate().await.expect("gate");
19682        assert!(runner.state.resource_block.is_none());
19683        assert!(
19684            !runner
19685                .state
19686                .events
19687                .iter()
19688                .any(|event| { event.message.contains("waiting for the shared build cache") }),
19689            "a gate with nothing to run must never request the held cache lease"
19690        );
19691        assert!(
19692            runner.state.gate_ran,
19693            "zero commands is still a real, immediate attempt"
19694        );
19695        assert!(runner.state.gate.is_empty());
19696        assert_ne!(
19697            runner.state.status,
19698            RunStatus::Blocked,
19699            "must not read as resource-blocked on a lease it never asked for"
19700        );
19701    }
19702
19703    /// A separate process carries a live worker's environment throughout all
19704    /// verification entry points; the parallel harness never changes its env.
19705    #[cfg(unix)]
19706    #[tokio::test]
19707    async fn verification_is_standalone_inside_a_live_worker() {
19708        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19709        if std::env::var_os(CHILD).is_none() {
19710            let home = tempfile::tempdir().unwrap();
19711            let generation = crate::worker::pin_current(home.path()).unwrap();
19712            let output = std::process::Command::new(std::env::current_exe().unwrap())
19713                .args([
19714                    "--exact",
19715                    "graph::tests::verification_is_standalone_inside_a_live_worker",
19716                    "--nocapture",
19717                ])
19718                .env(CHILD, "1")
19719                .env("MAGI_HOME", home.path())
19720                .env("MAGI_WORKER_HOME", home.path())
19721                .env("MAGI_WORKER_ID", "abcdef")
19722                .env("MAGI_WORKER_GENERATION", &generation.id)
19723                .env("MAGI_WORKER_EXE", &generation.executable)
19724                .env("MAGI_RUN", "verification-parent")
19725                .env("MAGI_NODE", "gate")
19726                .env("MAGI_SEAT", "worker-fixture")
19727                .env("MAGI_PROCESS_OWNER", "fixture-owner")
19728                .env("MAGI_NO_AUTOUPDATE", "0")
19729                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19730                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19731                .env(
19732                    "VERIFY_EXPECT_TOOLCHAIN",
19733                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19734                )
19735                .env(
19736                    "VERIFY_EXPECT_CACHE",
19737                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19738                )
19739                .output()
19740                .unwrap();
19741            assert!(
19742                output.status.success()
19743                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19744                "{}\n{}",
19745                String::from_utf8_lossy(&output.stdout),
19746                String::from_utf8_lossy(&output.stderr)
19747            );
19748            return;
19749        }
19750        let home = crate::run::home();
19751        let directory = home.join("workers/jobs/abcdef");
19752        std::fs::create_dir_all(&directory).unwrap();
19753        let job = crate::worker::Job {
19754            id: "abcdef".into(),
19755            sequence: 1,
19756            kind: "task".into(),
19757            key: "task:verification".into(),
19758            payload: serde_json::json!({}),
19759            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19760            version: Some(env!("CARGO_PKG_VERSION").into()),
19761            status: crate::worker::Status::Running,
19762            result: None,
19763            pid: Some(std::process::id()),
19764            created_at: 0,
19765            finished_at: None,
19766        };
19767        std::fs::write(
19768            directory.join("job.json"),
19769            serde_json::to_vec(&job).unwrap(),
19770        )
19771        .unwrap();
19772        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19773        let assert_worker = || {
19774            assert_eq!(
19775                crate::worker::current_job(&home).unwrap().unwrap().id,
19776                job.id
19777            );
19778            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19779            assert_eq!(
19780                crate::worker::helper_executable(&home).unwrap(),
19781                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19782            );
19783            assert!(
19784                crate::updater::Checker::new(&crate::config::Update {
19785                    mode: crate::config::UpdateMode::Notify,
19786                    interval: None,
19787                    auto_restart: false,
19788                })
19789                .is_none(),
19790                "the driver remains pinned, even while tests are standalone"
19791            );
19792        };
19793        assert_worker();
19794        let repo = home.join("repo");
19795        std::fs::create_dir_all(&repo).unwrap();
19796        init_repo(&repo);
19797        // Keep storage, seat/control context, containment and tool lookup.
19798        // No inherited values (or credentials) are printed on success.
19799        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
19800            test -z "${MAGI_WORKER_GENERATION+x}" &&
19801            test -z "${MAGI_WORKER_HOME+x}" &&
19802            test -z "${MAGI_WORKER_EXE+x}" &&
19803            test "$MAGI_HOME/repo" -ef . &&
19804            test "$MAGI_RUN" = verification-parent &&
19805            test "$MAGI_NODE" = gate &&
19806            test "$MAGI_SEAT" = worker-fixture &&
19807            test -n "$MAGI_PROCESS_OWNER" &&
19808            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
19809            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
19810            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
19811            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
19812        // Negative control: an ordinary child inherits the contamination.
19813        assert!(
19814            !std::process::Command::new("sh")
19815                .args(["-c", probe])
19816                .current_dir(&repo)
19817                .status()
19818                .unwrap()
19819                .success()
19820        );
19821        let unrelated_home = tempfile::tempdir().unwrap();
19822        assert!(
19823            crate::worker::current_job(unrelated_home.path()).is_err(),
19824            "an inherited worker id cannot be resolved in a unit fixture's home"
19825        );
19826        let mut runner = runner_at(RunStatus::Gating);
19827        runner.state.repo = repo.clone();
19828        runner.state.config.disk.min_free_bytes = 0;
19829        runner.state.config.coordination = crate::config::test_coordination();
19830        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
19831        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
19832        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
19833        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
19834        let winner = Candidate {
19835            index: 0,
19836            label: 'A',
19837            agent: "fixture".into(),
19838            branch: "main".into(),
19839            worktree: repo.clone(),
19840            summary: String::new(),
19841            stat: String::new(),
19842            files: 0,
19843            commits: 0,
19844            empty: false,
19845            failed: None,
19846            verified_noop: None,
19847            duration_ms: 0,
19848            folded: false,
19849        };
19850        let (e2e, retried, pids) = run_e2e_with_retry(
19851            &mut runner.state,
19852            &["sh".into(), "-c".into()],
19853            &[format!("{probe} && printf initial")],
19854            &repo,
19855            Duration::from_secs(10),
19856            "initial",
19857        )
19858        .await;
19859        assert!(!retried && pids.is_empty());
19860        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
19861        assert_eq!(e2e[0].output_tail, "initial");
19862        let gate = runner.run_gate(&winner).await.unwrap();
19863        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
19864        // An interrupted repair must recheck e2e from saved state. Landing
19865        // reuses that state and must run both pre-gate and gate again.
19866        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
19867            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
19868            passed: false,
19869            outcomes: Vec::new(),
19870        });
19871        runner.state.save().unwrap();
19872        let mut resumed = RunState::load(&runner.state.id).unwrap();
19873        let wait = crate::repo_lock::WaitPolicy::default();
19874        assert!(matches!(
19875            check_gate_repair_e2e(&mut resumed, &winner, &wait)
19876                .await
19877                .unwrap(),
19878            GateFix::Retry
19879        ));
19880        let saved = RunState::load(&resumed.id).unwrap();
19881        let repair = saved.gate_repair_e2e.unwrap();
19882        assert!(repair.passed);
19883        assert_eq!(repair.outcomes.len(), 1);
19884        assert!(repair.outcomes[0].ok());
19885        let landing = run_landing_gate(&mut resumed, &winner, &wait)
19886            .await
19887            .unwrap();
19888        assert_eq!(landing.len(), 1);
19889        assert!(landing[0].ok(), "{landing:?}");
19890        assert_eq!(
19891            std::fs::read_to_string(repo.join("phases")).unwrap(),
19892            "pre\ngate\ne2e\npre\ngate\n"
19893        );
19894        assert_worker();
19895    }
19896
19897    #[cfg(unix)]
19898    #[tokio::test]
19899    async fn verification_keeps_the_panic_and_the_full_output() {
19900        crate::run::pin_test_home();
19901        let tmp = tempfile::tempdir().unwrap();
19902        let mut config = Config::default();
19903        config.disk.min_free_bytes = 0;
19904        let mut state = RunState::new(
19905            tmp.path().to_owned(),
19906            "main".into(),
19907            "base".into(),
19908            "verification evidence".into(),
19909            config,
19910        );
19911        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
19912                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
19913                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
19914        let mut names = Vec::new();
19915        for _ in 0..2 {
19916            let (outcomes, _) = run_commands(
19917                &mut state,
19918                "gate",
19919                "gate",
19920                0,
19921                &["sh".into(), "-c".into()],
19922                &[script.into()],
19923                tmp.path(),
19924                Duration::from_secs(60),
19925            )
19926            .await;
19927            let tail = &outcomes[0].output_tail;
19928            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
19929            assert!(tail.contains("boom assertion"));
19930            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
19931            let line = tail.lines().next().unwrap();
19932            names.push(line.to_owned());
19933        }
19934        assert_ne!(names[0], names[1], "the second run must not overwrite");
19935        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
19936        assert!(full.len() > 500_000 && full.contains("boom assertion"));
19937        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
19938
19939        // Storage failures are evidence too; preserve the command's verdict.
19940        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
19941        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
19942        let (outcomes, _) = run_commands(
19943            &mut state,
19944            "gate",
19945            "gate",
19946            0,
19947            &["sh".into(), "-c".into()],
19948            &["echo failure-output; exit 7".into()],
19949            tmp.path(),
19950            Duration::from_secs(10),
19951        )
19952        .await;
19953        assert_eq!(outcomes[0].code, Some(7));
19954        assert!(state.events.iter().any(|e| {
19955            e.node == "gate" && e.message.contains("could not save the full gate output:")
19956        }));
19957    }
19958
19959    #[cfg(unix)]
19960    #[tokio::test]
19961    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
19962        crate::run::pin_test_home();
19963        let tmp = tempfile::tempdir().unwrap();
19964        let mut config = Config::default();
19965        config.disk.min_free_bytes = 0;
19966        let mut state = RunState::new(
19967            tmp.path().to_owned(),
19968            "main".into(),
19969            "base".into(),
19970            "verification evidence".into(),
19971            config,
19972        );
19973        let (outcomes, pids) = run_commands(
19974            &mut state,
19975            "gate",
19976            "gate",
19977            1,
19978            &["sh".into(), "-c".into()],
19979            &[
19980                "printf before-timeout; exec sleep 30".into(),
19981                "kill -TERM $$".into(),
19982                "exit 7".into(),
19983                "printf success".into(),
19984            ],
19985            tmp.path(),
19986            Duration::from_secs(1),
19987        )
19988        .await;
19989        wait_for_timed_out_children_to_die(&pids).await;
19990        assert_eq!(
19991            pids.len(),
19992            1,
19993            "unleased timeout retains its death-check PID"
19994        );
19995        assert_eq!(outcomes.len(), 4);
19996        assert_eq!(outcomes[0].code, None);
19997        assert!(outcomes[0].output_tail.contains("before-timeout"));
19998        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
19999        assert_eq!(outcomes[1].code, None);
20000        assert!(outcomes[1].output_tail.contains("command terminated:"));
20001        assert!(!outcomes[1].output_tail.contains("timed out"));
20002        assert_eq!(outcomes[2].code, Some(7));
20003        assert!(!outcomes[2].output_tail.contains("timed out"));
20004        assert_eq!(outcomes[3].code, Some(0));
20005        assert_eq!(outcomes[3].output_tail, "success");
20006        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20007    }
20008
20009    /// The addendum's second gap: a `verify.gate` command running for real
20010    /// wall-clock time had nothing at all to show for it in `active` before
20011    /// `run_commands` learned to record it — a run could sit in `Gating` for
20012    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20013    /// what was actually happening. Proven with a genuinely still-running
20014    /// command, not just a before/after check on the final state: a poller
20015    /// task reads the same `run.json` `gate()` is writing, the same way the
20016    /// phone or `magi show` would, while the shell command is still blocked
20017    /// on its own release marker.
20018    #[tokio::test]
20019    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20020        crate::run::pin_test_home();
20021
20022        let tmp = tempfile::tempdir().expect("tempdir");
20023        let repo = tmp.path().join("repo");
20024        std::fs::create_dir_all(&repo).unwrap();
20025        init_repo(&repo);
20026
20027        let mut config = Config::default();
20028        config.disk.min_free_bytes = 0;
20029        config.verify.gate = vec![
20030            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20031             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20032                .to_owned(),
20033        ];
20034
20035        let mut state = RunState::new(
20036            repo.clone(),
20037            "main".to_owned(),
20038            "deadbeef".to_owned(),
20039            "task".to_owned(),
20040            config,
20041        );
20042        let run_id = state.id.clone();
20043        state.candidates = vec![Candidate {
20044            index: 0,
20045            label: 'A',
20046            agent: "alpha".to_owned(),
20047            branch: "does-not-exist".to_owned(),
20048            worktree: repo.clone(),
20049            summary: String::new(),
20050            stat: String::new(),
20051            files: 0,
20052            commits: 0,
20053            empty: false,
20054            failed: None,
20055            verified_noop: None,
20056            duration_ms: 0,
20057            folded: false,
20058        }];
20059        state.tally = Some(Tally {
20060            first_choice: BTreeMap::from([('A', 1)]),
20061            borda: BTreeMap::new(),
20062            winner: 'A',
20063            rankings: 1,
20064            unanimous_initial: true,
20065            deliberated: false,
20066            changed_votes: 0,
20067            unanimous_final: true,
20068            tie_break: None,
20069            judges: 0,
20070            present: 0,
20071            quorum: 0,
20072            met_quorum: true,
20073            uncontested: Some("only candidate A produced a change".to_owned()),
20074        });
20075        state.reviews = vec![ReviewRound {
20076            round: 1,
20077            head: "deadbeef".to_owned(),
20078            verified_head: None,
20079            verified_at: None,
20080            reviews: Vec::new(),
20081            e2e: Vec::new(),
20082            fix: None,
20083            blocking: 0,
20084            answered: 0,
20085            expected: 0,
20086            clean: true,
20087            verify_retried: false,
20088            e2e_deferred: false,
20089            e2e_defer_reason: None,
20090            progressed: false,
20091            vote_split: false,
20092            reconsideration: Vec::new(),
20093            verdict: None,
20094        }];
20095
20096        let mut runner = Runner {
20097            state,
20098            roles: ResolvedRoles {
20099                implementers: Vec::new(),
20100                judges: Vec::new(),
20101                reviewers: Vec::new(),
20102                fixer: None,
20103                conductor: conductor(),
20104                implementer_roster: Vec::new(),
20105                judge_roster: Vec::new(),
20106                reviewer_roster: Vec::new(),
20107            },
20108            sem: Arc::new(Semaphore::new(1)),
20109            pause: Pause::new(),
20110            interrupt: Pause::new(),
20111        };
20112
20113        let started_marker = repo.join("started.marker");
20114        let release_marker = repo.join("release.marker");
20115        let poller = tokio::spawn(async move {
20116            // Bounded so a regression that never records the task entry
20117            // fails this test in seconds instead of hanging the suite —
20118            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20119            // does_not_cut_it_short` uses for the same reason.
20120            for _ in 0..100 {
20121                if started_marker.exists()
20122                    && let Ok(s) = crate::run::RunState::load(&run_id)
20123                    && let Some(a) = s.active.get("gate")
20124                {
20125                    std::fs::write(&release_marker, b"go").expect("release marker");
20126                    return Some(a.clone());
20127                }
20128                tokio::time::sleep(Duration::from_millis(50)).await;
20129            }
20130            None
20131        });
20132
20133        runner.gate().await.expect("gate");
20134        let captured = poller.await.expect("poller task");
20135        let captured = captured.expect(
20136            "the poller never saw a `gate` task entry in run.json while the command was \
20137             still blocked on its own release marker",
20138        );
20139
20140        assert_eq!(captured.task.as_deref(), Some("gate"));
20141        assert_eq!(captured.node, "gate");
20142        assert_eq!(captured.index, Some(1));
20143        assert_eq!(captured.total, Some(1));
20144        assert!(
20145            captured
20146                .command
20147                .as_deref()
20148                .is_some_and(|c| c.contains("started.marker")),
20149            "{captured:?}"
20150        );
20151
20152        assert!(
20153            runner.state.active.is_empty(),
20154            "the entry must be cleared once the command actually finished: {:?}",
20155            runner.state.active
20156        );
20157        assert!(runner.state.gate_ran);
20158        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20159    }
20160
20161    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20162    /// record for `land`; one with only a Minor, or no reject, leaves none.
20163    #[tokio::test]
20164    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20165        use crate::verdict::{Finding, ReviewVote, Severity};
20166        crate::run::pin_test_home();
20167        let tmp = tempfile::tempdir().expect("tempdir");
20168        let repo = tmp.path().join("repo");
20169        std::fs::create_dir_all(&repo).unwrap();
20170        init_repo(&repo);
20171
20172        for (severity, vote, expect) in [
20173            (Severity::Major, ReviewVote::Reject, true),
20174            (Severity::Minor, ReviewVote::Reject, false),
20175            (Severity::Major, ReviewVote::Approve, false),
20176        ] {
20177            let mut round = review_round(false, 1, 1, 1, false, true);
20178            round.reviews = vec![ReviewRecord {
20179                reviewer: 1,
20180                agent: "alpha".to_owned(),
20181                summary: String::new(),
20182                findings: vec![Finding {
20183                    id: "R1-1-1".to_owned(),
20184                    severity,
20185                    file: None,
20186                    line: None,
20187                    title: "t".to_owned(),
20188                    detail: String::new(),
20189                }],
20190                vote: Some(vote),
20191                failed: None,
20192                duration_ms: 0,
20193                attempts: 0,
20194            }];
20195            let mut state = RunState::new(
20196                repo.clone(),
20197                "main".to_owned(),
20198                "deadbeef".to_owned(),
20199                "task".to_owned(),
20200                Config::default(),
20201            );
20202            state.reviews = vec![round];
20203            let mut runner = Runner {
20204                state,
20205                roles: ResolvedRoles {
20206                    implementers: Vec::new(),
20207                    judges: Vec::new(),
20208                    reviewers: Vec::new(),
20209                    fixer: None,
20210                    conductor: conductor(),
20211                    implementer_roster: Vec::new(),
20212                    judge_roster: Vec::new(),
20213                    reviewer_roster: Vec::new(),
20214                },
20215                sem: Arc::new(Semaphore::new(1)),
20216                pause: Pause::new(),
20217                interrupt: Pause::new(),
20218            };
20219            let shell = runner.state.config.shell();
20220            runner
20221                .stop_reviewing("round budget spent", &shell, &repo)
20222                .await
20223                .expect("stop_reviewing");
20224            assert_eq!(runner.state.status, RunStatus::Gating);
20225            assert_eq!(
20226                runner.state.contested_handoff.is_some(),
20227                expect,
20228                "{severity:?} + {vote:?}"
20229            );
20230        }
20231    }
20232
20233    /// The shape the incident this whole fix responds to actually had: the
20234    /// round budget spent, the last round's own e2e blocked on the shared
20235    /// build cache (held here by a live pid — this test process — exactly
20236    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20237    /// without forking a process). `stop_reviewing` must retry it — not
20238    /// silently leave the round looking untouched (the catch-up-only half of
20239    /// the bug), and not read the contention as a red `e2e` and block the
20240    /// run on it (the other half). Called directly, the same way
20241    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20242    /// above exercises `gate`, so this never needs a real cargo build to
20243    /// reach: the lease is never released, so `with_cache_lease` never gets
20244    /// past acquiring it into anything that would need a real workspace.
20245    #[tokio::test]
20246    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20247        crate::run::pin_test_home();
20248        let home = crate::run::home();
20249
20250        let tmp = tempfile::tempdir().expect("tempdir");
20251        let repo = tmp.path().join("repo");
20252        std::fs::create_dir_all(&repo).unwrap();
20253        init_repo(&repo);
20254        let head = crate::git::rev_parse(&repo, "HEAD")
20255            .await
20256            .expect("rev-parse");
20257        // Unique to this test, so holding its lease cannot collide with
20258        // another test sharing the same process-wide `home`.
20259        let cache_dir = tmp.path().join("target");
20260
20261        let mut config = Config::default();
20262        config.verify.e2e = vec![format!(
20263            "CARGO_TARGET_DIR='{}' test -f README.md",
20264            cache_dir.display()
20265        )];
20266        config.graph.review_rounds = 1;
20267        // Bounded so a regression that does start waiting fails the test in
20268        // seconds, not hangs it.
20269        config.graph.timeout_verify = Some(2);
20270        config.graph.timeout_resource_wait = 2;
20271
20272        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20273        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20274            .expect("no io error acquiring directly")
20275        {
20276            crate::cache::AcquireOutcome::Acquired(g) => g,
20277            crate::cache::AcquireOutcome::Busy(b) => {
20278                panic!("expected the direct acquire to win the lease first: {b:?}")
20279            }
20280        };
20281
20282        let mut state = RunState::new(
20283            repo.clone(),
20284            "main".to_owned(),
20285            head.clone(),
20286            "task".to_owned(),
20287            config,
20288        );
20289        state.candidates = vec![Candidate {
20290            index: 0,
20291            label: 'A',
20292            agent: "alpha".to_owned(),
20293            branch: "does-not-exist".to_owned(),
20294            worktree: repo.clone(),
20295            summary: String::new(),
20296            stat: String::new(),
20297            files: 0,
20298            commits: 0,
20299            empty: false,
20300            failed: None,
20301            verified_noop: None,
20302            duration_ms: 0,
20303            folded: false,
20304        }];
20305        state.tally = Some(Tally {
20306            first_choice: BTreeMap::from([('A', 1)]),
20307            borda: BTreeMap::new(),
20308            winner: 'A',
20309            rankings: 1,
20310            unanimous_initial: true,
20311            deliberated: false,
20312            changed_votes: 0,
20313            unanimous_final: true,
20314            tie_break: None,
20315            judges: 0,
20316            present: 0,
20317            quorum: 0,
20318            met_quorum: true,
20319            uncontested: Some("only candidate A produced a change".to_owned()),
20320        });
20321        // The round budget's last round, deferred: `needs_catchup_run`'s
20322        // other trigger. `stop_reviewing`'s retry machinery must treat this
20323        // exactly like a resource-blocked attempt once it actually runs.
20324        state.reviews = vec![ReviewRound {
20325            round: 1,
20326            head: head.clone(),
20327            verified_head: None,
20328            verified_at: None,
20329            reviews: Vec::new(),
20330            e2e: Vec::new(),
20331            fix: None,
20332            blocking: 1,
20333            answered: 1,
20334            expected: 1,
20335            clean: false,
20336            verify_retried: false,
20337            e2e_deferred: true,
20338            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20339            progressed: false,
20340            vote_split: false,
20341            reconsideration: Vec::new(),
20342            verdict: None,
20343        }];
20344
20345        let mut runner = Runner {
20346            state,
20347            roles: ResolvedRoles {
20348                implementers: Vec::new(),
20349                judges: Vec::new(),
20350                reviewers: Vec::new(),
20351                fixer: None,
20352                conductor: conductor(),
20353                implementer_roster: Vec::new(),
20354                judge_roster: Vec::new(),
20355                reviewer_roster: Vec::new(),
20356            },
20357            sem: Arc::new(Semaphore::new(1)),
20358            pause: Pause::new(),
20359            interrupt: Pause::new(),
20360        };
20361
20362        let shell = runner.state.config.shell();
20363        runner
20364            .stop_reviewing("round budget spent", &shell, &repo)
20365            .await
20366            .expect("stop_reviewing");
20367
20368        let last = runner.state.reviews.last().expect("round record");
20369        assert_eq!(
20370            last.e2e_status(),
20371            E2eStatus::ResourceBlocked,
20372            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20373             failed: {last:?}"
20374        );
20375        assert_eq!(
20376            last.verified_head.as_deref(),
20377            Some(head.as_str()),
20378            "which commit this attempt targeted is known even though nothing finished checking \
20379             it"
20380        );
20381        let first_attempt_at = last
20382            .verified_at
20383            .expect("when this attempt ran is known too");
20384        assert_ne!(
20385            runner.state.status,
20386            RunStatus::Blocked,
20387            "contention is evidence about the machine, not the patch — it must not settle the \
20388             run as blocked: {:?}",
20389            runner.state.status
20390        );
20391        assert!(
20392            !runner
20393                .state
20394                .events
20395                .iter()
20396                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20397            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20398            runner.state.events
20399        );
20400
20401        // The cache is still held: a later reentry must retry the same
20402        // round's verification again — not leave it looking exactly as
20403        // untouched as the first blocked attempt, which is indistinguishable
20404        // from never having tried again at all.
20405        runner
20406            .stop_reviewing("round budget spent", &shell, &repo)
20407            .await
20408            .expect("stop_reviewing retry");
20409        assert_eq!(
20410            runner.state.reviews.len(),
20411            1,
20412            "no new round was started: {:?}",
20413            runner.state.reviews
20414        );
20415        let last = runner.state.reviews.last().expect("round record");
20416        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20417        assert!(
20418            last.verified_at.expect("still known") > first_attempt_at,
20419            "a second reentry must be a fresh attempt, not a stale copy of the first"
20420        );
20421        assert_ne!(runner.state.status, RunStatus::Blocked);
20422
20423        held.release();
20424    }
20425
20426    #[cfg(unix)]
20427    #[tokio::test]
20428    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20429        ask_test_home();
20430        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20431            let tmp = tempfile::tempdir().unwrap();
20432            let repo = tmp.path().join("repo");
20433            std::fs::create_dir(&repo).unwrap();
20434            init_repo(&repo);
20435            let run_git = |args: &[&str]| {
20436                let out = std::process::Command::new("git")
20437                    .args(args)
20438                    .current_dir(&repo)
20439                    .output()
20440                    .unwrap();
20441                assert!(
20442                    out.status.success(),
20443                    "{args:?}: {}",
20444                    String::from_utf8_lossy(&out.stderr)
20445                );
20446            };
20447            let origin = tmp.path().join("origin.git");
20448            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20449            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20450            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20451            run_git(&["add", "spec.rs"]);
20452            run_git(&["commit", "-m", "initial spec"]);
20453            run_git(&["push", "origin", "main"]);
20454            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20455            run_git(&["checkout", "-b", "fix/review"]);
20456            std::fs::write(
20457                repo.join("check.rs"),
20458                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20459            )
20460            .unwrap();
20461            run_git(&["add", "check.rs"]);
20462            run_git(&["commit", "-m", "use spec"]);
20463            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20464            run_git(&["checkout", "main"]);
20465            std::fs::write(
20466                repo.join("spec.rs"),
20467                "struct AgentSpec { max_turn_secs: u64 }\n",
20468            )
20469            .unwrap();
20470            run_git(&["add", "spec.rs"]);
20471            run_git(&["commit", "-m", "extend spec"]);
20472            run_git(&["push", "origin", "main"]);
20473            run_git(&["checkout", "fix/review"]);
20474
20475            let mut runner = runner_at(RunStatus::Reviewing);
20476            runner.state.repo = repo.clone();
20477            runner.state.base_commit = base;
20478            runner.state.pr_head = Some("fix/review".into());
20479            runner.state.config.graph.review_rounds = max_rounds;
20480            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20481            runner.state.config.graph.sessions = false;
20482            runner.state.config.disk.min_free_bytes = 0;
20483            runner.state.config.verify.e2e =
20484                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20485            runner.state.candidates = vec![Candidate {
20486                index: 0,
20487                label: 'A',
20488                agent: "fixture".into(),
20489                branch: "fix/review".into(),
20490                worktree: repo.clone(),
20491                summary: String::new(),
20492                stat: String::new(),
20493                files: 1,
20494                commits: 1,
20495                empty: false,
20496                failed: None,
20497                verified_noop: None,
20498                duration_ms: 0,
20499                folded: false,
20500            }];
20501            runner.state.tally = Some(Tally {
20502                first_choice: BTreeMap::new(),
20503                borda: BTreeMap::new(),
20504                winner: 'A',
20505                rankings: 0,
20506                unanimous_initial: true,
20507                deliberated: false,
20508                changed_votes: 0,
20509                unanimous_final: true,
20510                tie_break: None,
20511                judges: 0,
20512                present: 0,
20513                quorum: 0,
20514                met_quorum: true,
20515                uncontested: Some("review-only".into()),
20516            });
20517            let mut last = review_round(false, 0, 1, 1, false, false);
20518            last.head = reviewed.clone();
20519            last.e2e_deferred = true;
20520            last.e2e[0].resource_blocked = true;
20521            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20522            if max_rounds == 1 {
20523                last.e2e.clear();
20524            }
20525            runner.state.reviews = vec![last];
20526            let calls = tmp.path().join("fix-calls");
20527            let mut fixer = spec("repair");
20528            fixer
20529                .env
20530                .insert("FIX_CALLS".into(), calls.display().to_string());
20531            fixer.command = vec!["sh".into(), "-c".into(), r#"
20532                echo fix >> "$FIX_CALLS"
20533                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20534                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20535                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20536                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20537            "#.into()];
20538            let mut reviewer = spec("reviewer");
20539            reviewer.command = vec![
20540                "sh".into(),
20541                "-c".into(),
20542                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20543                    .into(),
20544            ];
20545            runner.roles.fixer = Some(vec![fixer]);
20546            runner.roles.reviewers = vec![reviewer];
20547
20548            runner.sync_to_base().await.unwrap();
20549            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20550            assert_ne!(rebased, reviewed);
20551            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20552            if reentry {
20553                // Simulate a crash after catch-up evidence was saved but before
20554                // dispatching the fixer. No dedicated pending-repair field exists.
20555                let shell = runner.state.config.shell();
20556                runner
20557                    .stop_unresolved_review("deferred verification", &shell, &repo)
20558                    .await
20559                    .unwrap();
20560                assert!(runner.catchup_repairable());
20561                assert!(!calls.exists());
20562                runner.state = RunState::load(&runner.state.id).unwrap();
20563                assert!(runner.catchup_repairable());
20564            }
20565            runner.review_loop().await.unwrap();
20566            let first = &runner.state.reviews[0];
20567            assert_eq!(first.head, reviewed);
20568            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20569            assert!(
20570                first.verify_retried,
20571                "a reproduced build failure gets exactly one retry: {:?}",
20572                first.e2e
20573            );
20574            assert!(first.e2e[0].output_tail.contains("E0063"));
20575            assert!(!runner.catchup_repairable());
20576            if max_rounds == 2 {
20577                assert_eq!(
20578                    runner.state.status,
20579                    RunStatus::Gating,
20580                    "{}",
20581                    crate::report::run(&runner.state)
20582                );
20583                assert_eq!(runner.state.reviews.len(), 2);
20584                assert!(first.fix.as_ref().unwrap().committed);
20585                assert!(runner.state.reviews[1].clean);
20586                assert_eq!(
20587                    runner
20588                        .state
20589                        .reviews
20590                        .iter()
20591                        .filter(|r| r.fix.is_some())
20592                        .count(),
20593                    1
20594                );
20595                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20596                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20597            } else {
20598                assert_eq!(runner.state.status, RunStatus::Blocked);
20599                assert!(first.fix.is_none());
20600                assert!(!calls.exists());
20601                let recovery = crate::recovery::assess(
20602                    "prior",
20603                    Some(&runner.state),
20604                    &runner.landing_base(),
20605                    crate::recovery::Relation::Diverged,
20606                );
20607                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20608                let reasons = recovery.reasons.join("\n");
20609                assert!(
20610                    reasons.contains(&reviewed) && reasons.contains(&rebased),
20611                    "{reasons}"
20612                );
20613                assert!(
20614                    reasons.contains("base sync to")
20615                        && reasons.contains("additional review is required"),
20616                    "{reasons}"
20617                );
20618            }
20619        }
20620    }
20621
20622    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
20623    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
20624    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
20625    /// own top-of-function fast path (`review_conclusion`) correctly reads
20626    /// this shape as `None` rather than guessing `Blocked`, and the loop's
20627    /// own `for` range is empty once the round budget is spent, so
20628    /// `review_loop` must retry the check itself rather than silently doing
20629    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
20630    /// above exercises directly, but through `review_loop`'s own entry point
20631    /// this time, proving the wiring between the two rather than just the
20632    /// retry logic in isolation.
20633    #[tokio::test]
20634    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20635        crate::run::pin_test_home();
20636        let home = crate::run::home();
20637
20638        let tmp = tempfile::tempdir().expect("tempdir");
20639        let repo = tmp.path().join("repo");
20640        std::fs::create_dir_all(&repo).unwrap();
20641        init_repo(&repo);
20642        let head = crate::git::rev_parse(&repo, "HEAD")
20643            .await
20644            .expect("rev-parse");
20645        let cache_dir = tmp.path().join("target");
20646
20647        let mut config = Config::default();
20648        config.verify.e2e = vec![format!(
20649            "CARGO_TARGET_DIR='{}' test -f README.md",
20650            cache_dir.display()
20651        )];
20652        config.graph.review_rounds = 1;
20653        config.graph.timeout_verify = Some(2);
20654        config.graph.timeout_resource_wait = 2;
20655
20656        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20657        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20658            .expect("no io error acquiring directly")
20659        {
20660            crate::cache::AcquireOutcome::Acquired(g) => g,
20661            crate::cache::AcquireOutcome::Busy(b) => {
20662                panic!("expected the direct acquire to win the lease first: {b:?}")
20663            }
20664        };
20665
20666        let mut state = RunState::new(
20667            repo.clone(),
20668            "main".to_owned(),
20669            head.clone(),
20670            "task".to_owned(),
20671            config,
20672        );
20673        state.candidates = vec![Candidate {
20674            index: 0,
20675            label: 'A',
20676            agent: "alpha".to_owned(),
20677            branch: "does-not-exist".to_owned(),
20678            worktree: repo.clone(),
20679            summary: String::new(),
20680            stat: String::new(),
20681            files: 0,
20682            commits: 0,
20683            empty: false,
20684            failed: None,
20685            verified_noop: None,
20686            duration_ms: 0,
20687            folded: false,
20688        }];
20689        state.tally = Some(Tally {
20690            first_choice: BTreeMap::from([('A', 1)]),
20691            borda: BTreeMap::new(),
20692            winner: 'A',
20693            rankings: 1,
20694            unanimous_initial: true,
20695            deliberated: false,
20696            changed_votes: 0,
20697            unanimous_final: true,
20698            tie_break: None,
20699            judges: 0,
20700            present: 0,
20701            quorum: 0,
20702            met_quorum: true,
20703            uncontested: Some("only candidate A produced a change".to_owned()),
20704        });
20705        // The exact shape a prior process's `stop_reviewing` would have left
20706        // on disk: the round budget's last round, a real attempt already
20707        // made and already resource-blocked.
20708        state.reviews = vec![ReviewRound {
20709            round: 1,
20710            head: head.clone(),
20711            verified_head: Some(head.clone()),
20712            verified_at: Some(jiff::Timestamp::now()),
20713            reviews: Vec::new(),
20714            e2e: vec![CommandOutcome {
20715                command: format!(
20716                    "CARGO_TARGET_DIR='{}' test -f README.md",
20717                    cache_dir.display()
20718                ),
20719                code: None,
20720                output_tail: "waiting for the shared build cache".to_owned(),
20721                duration_ms: 0,
20722                resource_blocked: true,
20723            }],
20724            fix: None,
20725            blocking: 1,
20726            answered: 1,
20727            expected: 1,
20728            clean: false,
20729            verify_retried: false,
20730            e2e_deferred: false,
20731            e2e_defer_reason: None,
20732            progressed: false,
20733            vote_split: false,
20734            reconsideration: Vec::new(),
20735            verdict: None,
20736        }];
20737
20738        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20739        let mut runner = Runner {
20740            state,
20741            roles: ResolvedRoles {
20742                implementers: Vec::new(),
20743                judges: Vec::new(),
20744                reviewers: Vec::new(),
20745                fixer: None,
20746                conductor: conductor(),
20747                implementer_roster: Vec::new(),
20748                judge_roster: Vec::new(),
20749                reviewer_roster: Vec::new(),
20750            },
20751            sem: Arc::new(Semaphore::new(1)),
20752            pause: Pause::new(),
20753            interrupt: Pause::new(),
20754        };
20755
20756        // The lease is still held throughout, so this reentry's own retry is
20757        // also contended — proving `review_loop` actually tried again (not
20758        // that it happened to succeed) is what the timestamp comparison
20759        // below is for.
20760        runner.review_loop().await.expect("review_loop");
20761
20762        assert_eq!(
20763            runner.state.reviews.len(),
20764            1,
20765            "no new round was started on top of the unresolved one: {:?}",
20766            runner.state.reviews
20767        );
20768        let last = &runner.state.reviews[0];
20769        assert_eq!(
20770            last.e2e_status(),
20771            E2eStatus::ResourceBlocked,
20772            "still contended: {last:?}"
20773        );
20774        assert!(
20775            last.verified_at.expect("still known") > first_attempt_at,
20776            "review_loop must have actually retried the check, not left it exactly as found"
20777        );
20778        assert_ne!(
20779            runner.state.status,
20780            RunStatus::Blocked,
20781            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20782            runner.state.status
20783        );
20784
20785        held.release();
20786    }
20787
20788    #[tokio::test]
20789    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20790        crate::run::pin_test_home();
20791        for status in [RunStatus::Blocked, RunStatus::Gating] {
20792            let mut runner = runner_at(status);
20793            runner.state.config.graph.land = true;
20794            runner.state.gate_ran = true;
20795            runner.state.merge = Some(MergeOutcome {
20796                mode: MergeMode::Pr,
20797                ok: false,
20798                detail: "the fixer produced no commit".into(),
20799                empty: false,
20800            });
20801            let url = "https://example.invalid/o/r/pull/1";
20802            runner.state.pr = Some(crate::run::PrRecord {
20803                url: url.into(),
20804                number: 1,
20805                state: "open".into(),
20806                checks: "red".into(),
20807                round: 1,
20808                rounds: 1,
20809                red_at_merge: Vec::new(),
20810            });
20811            runner.state.save().unwrap();
20812            // The nonexistent repository makes observation fail before a forge
20813            // command can run. We still must reach land, rather than silently
20814            // return from the graph with its old gate and merge records.
20815            runner.execute_graph().await.unwrap();
20816            assert_eq!(runner.state.status, RunStatus::Blocked);
20817            assert!(runner.state.land_sync_pending);
20818            assert!(runner.state.events.iter().any(|event|
20819                event.node == "land" && event.message == format!("watching {url}")));
20820            assert_eq!(
20821                runner.state.merge.as_ref().unwrap().detail,
20822                "the fixer produced no commit"
20823            );
20824        }
20825    }
20826
20827    #[tokio::test]
20828    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
20829        crate::run::pin_test_home();
20830        let tmp = tempfile::tempdir().expect("tempdir");
20831        let repo = tmp.path().join("repo");
20832        std::fs::create_dir_all(&repo).unwrap();
20833        init_repo(&repo);
20834
20835        let mut config = Config::default();
20836        config.merge.mode = MergeMode::Pr;
20837        config.graph.land = true;
20838        config.graph.land_approval = false;
20839
20840        let mut state = RunState::new(
20841            repo.clone(),
20842            "main".to_owned(),
20843            "deadbeef".to_owned(),
20844            "task".to_owned(),
20845            config,
20846        );
20847        state.candidates = vec![Candidate {
20848            index: 0,
20849            label: 'A',
20850            agent: "alpha".to_owned(),
20851            branch: "does-not-exist".to_owned(),
20852            worktree: repo.clone(),
20853            summary: String::new(),
20854            stat: String::new(),
20855            files: 0,
20856            commits: 0,
20857            empty: false,
20858            failed: None,
20859            verified_noop: None,
20860            duration_ms: 0,
20861            folded: false,
20862        }];
20863        state.tally = Some(Tally {
20864            first_choice: BTreeMap::from([('A', 1)]),
20865            borda: BTreeMap::new(),
20866            winner: 'A',
20867            rankings: 1,
20868            unanimous_initial: true,
20869            deliberated: false,
20870            changed_votes: 0,
20871            unanimous_final: true,
20872            tie_break: None,
20873            judges: 0,
20874            present: 0,
20875            quorum: 0,
20876            met_quorum: true,
20877            uncontested: Some("only candidate A produced a change".to_owned()),
20878        });
20879        state.reviews = vec![ReviewRound {
20880            round: 1,
20881            head: "deadbeef".to_owned(),
20882            verified_head: None,
20883            verified_at: None,
20884            reviews: Vec::new(),
20885            e2e: Vec::new(),
20886            fix: None,
20887            blocking: 0,
20888            answered: 0,
20889            expected: 0,
20890            clean: true,
20891            verify_retried: false,
20892            e2e_deferred: false,
20893            e2e_defer_reason: None,
20894            progressed: false,
20895            vote_split: false,
20896            reconsideration: Vec::new(),
20897            verdict: None,
20898        }];
20899        state.gate = vec![CommandOutcome {
20900            command: "test".to_owned(),
20901            code: Some(0),
20902            output_tail: String::new(),
20903            duration_ms: 0,
20904            resource_blocked: false,
20905        }];
20906        state.gate_ran = true;
20907        // A first pass through `merge` already pushed and opened this pull
20908        // request; `status` is `Landing` because a previous call into `land`
20909        // parked or was interrupted before it reached a terminal outcome.
20910        state.status = RunStatus::Landing;
20911        state.merge = Some(MergeOutcome {
20912            mode: MergeMode::Pr,
20913            ok: true,
20914            detail: "https://example.invalid/x/y/pull/1".to_owned(),
20915            empty: false,
20916        });
20917
20918        // The Landing-resume shortcut calls `run_land` directly rather than
20919        // through `merge`, which is exactly the call site that used to skip
20920        // `settle_questions` - see the fixture below.
20921        ask_test_home();
20922        // A real resume loads a persisted run; driver ownership is checked
20923        // against that record before the Landing shortcut is entered.
20924        state.save().expect("save landing run");
20925        let store = ask::Questions::open();
20926        let q = ask_open_question(&store, &state.id);
20927
20928        let mut runner = Runner {
20929            state,
20930            roles: ResolvedRoles {
20931                implementers: Vec::new(),
20932                judges: Vec::new(),
20933                reviewers: Vec::new(),
20934                fixer: None,
20935                conductor: conductor(),
20936                implementer_roster: Vec::new(),
20937                judge_roster: Vec::new(),
20938                reviewer_roster: Vec::new(),
20939            },
20940            sem: Arc::new(Semaphore::new(1)),
20941            pause: Pause::new(),
20942            interrupt: Pause::new(),
20943        };
20944
20945        // `execute`, not `merge` directly: the Landing-resume shortcut lives
20946        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
20947        // exactly because `review_loop` would otherwise clobber the marker
20948        // first.
20949        runner.execute().await.expect("execute");
20950
20951        assert_eq!(
20952            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
20953            Some("https://example.invalid/x/y/pull/1"),
20954            "reentry must not push again or open a second pull request over the \
20955             one `land` is already watching"
20956        );
20957        assert_ne!(
20958            runner.state.status,
20959            RunStatus::Landing,
20960            "land could not actually reach the fake pull request, so it must \
20961             have given up rather than left the run silently parked forever"
20962        );
20963        // `land` could not reach the fake pull request, so it gave up into
20964        // `Blocked` - still resumable, so the question must not have been
20965        // swept just because this branch now also calls `settle_questions`.
20966        assert_eq!(runner.state.status, RunStatus::Blocked);
20967        assert!(
20968            store.get(&q.id).unwrap().status.open(),
20969            "Blocked is still alive; settle_questions must have been a no-op here"
20970        );
20971    }
20972
20973    fn state_with_round(round: ReviewRound) -> RunState {
20974        let mut s = RunState::new(
20975            PathBuf::from("/repo"),
20976            "main".to_owned(),
20977            "abc1234".to_owned(),
20978            "add retries".to_owned(),
20979            Config::default(),
20980        );
20981        s.reviews = vec![round];
20982        s
20983    }
20984
20985    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
20986        crate::verdict::Finding {
20987            id: id.to_owned(),
20988            severity,
20989            file: None,
20990            line: None,
20991            title: title.to_owned(),
20992            detail: String::new(),
20993        }
20994    }
20995
20996    #[test]
20997    fn pr_body_names_open_findings_and_declined_ones() {
20998        let round = ReviewRound {
20999            round: 2,
21000            head: "deadbee".to_owned(),
21001            verified_head: None,
21002            verified_at: None,
21003            reviews: vec![ReviewRecord {
21004                attempts: 0,
21005                reviewer: 1,
21006                agent: "alpha".to_owned(),
21007                summary: String::new(),
21008                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21009                vote: None,
21010                failed: None,
21011                duration_ms: 0,
21012            }],
21013            e2e: vec![CommandOutcome {
21014                command: "cargo test".to_owned(),
21015                code: Some(0),
21016                output_tail: String::new(),
21017                duration_ms: 0,
21018                resource_blocked: false,
21019            }],
21020            verify_retried: false,
21021            e2e_deferred: false,
21022            e2e_defer_reason: None,
21023            fix: Some(FixRecord {
21024                agent: "alpha".to_owned(),
21025                addressed: Vec::new(),
21026                rejected: vec![crate::verdict::Rejection {
21027                    id: "R1-1-1".to_owned(),
21028                    why: "not reachable from any caller".to_owned(),
21029                }],
21030                notes: String::new(),
21031                committed: true,
21032                failed: None,
21033                duration_ms: 0,
21034                continuation: None,
21035            }),
21036            blocking: 0,
21037            answered: 1,
21038            expected: 1,
21039            clean: false,
21040            progressed: true,
21041            vote_split: false,
21042            reconsideration: Vec::new(),
21043            verdict: None,
21044        };
21045        let state = state_with_round(round);
21046        let body = pr_message(&state, 'A').body;
21047
21048        assert!(body.contains("add retries"), "the task must still be there");
21049        assert!(body.contains("R2-1-1"), "{body}");
21050        assert!(!body.contains("unused import"), "{body}");
21051        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21052        assert!(
21053            !body.contains("not reachable from any caller"),
21054            "the reason it was declined: {body}"
21055        );
21056    }
21057
21058    #[test]
21059    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21060        let round = ReviewRound {
21061            round: 1,
21062            head: "deadbee".to_owned(),
21063            verified_head: None,
21064            verified_at: None,
21065            reviews: vec![ReviewRecord {
21066                attempts: 0,
21067                reviewer: 1,
21068                agent: "alpha".to_owned(),
21069                summary: String::new(),
21070                findings: Vec::new(),
21071                vote: None,
21072                failed: None,
21073                duration_ms: 0,
21074            }],
21075            e2e: Vec::new(),
21076            verify_retried: false,
21077            e2e_deferred: false,
21078            e2e_defer_reason: None,
21079            fix: None,
21080            blocking: 0,
21081            answered: 1,
21082            expected: 1,
21083            clean: true,
21084            progressed: false,
21085            vote_split: false,
21086            reconsideration: Vec::new(),
21087            verdict: None,
21088        };
21089        let state = state_with_round(round);
21090        let body = pr_message(&state, 'A').body;
21091        assert!(!body.contains("Open review findings"), "{body}");
21092        assert!(!body.contains("Declined"), "{body}");
21093    }
21094
21095    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21096        let mut state = RunState::new(
21097            PathBuf::from("/repo"),
21098            "main".to_owned(),
21099            "abc1234".to_owned(),
21100            instruction.to_owned(),
21101            Config::default(),
21102        );
21103        // These fixtures test execution flow, independent of host disk capacity.
21104        state.config.disk.min_free_bytes = 0;
21105        state.candidates.push(Candidate {
21106            index: 0,
21107            label: 'A',
21108            agent: "alpha".to_owned(),
21109            branch: "magi/x/A".to_owned(),
21110            worktree: PathBuf::from("/wt"),
21111            summary: summary.to_owned(),
21112            stat: String::new(),
21113            files: 1,
21114            commits: 1,
21115            empty: false,
21116            failed: None,
21117            verified_noop: None,
21118            folded: false,
21119            duration_ms: 0,
21120        });
21121        state
21122    }
21123
21124    fn review_state(subjects: &[&str]) -> RunState {
21125        let mut state = state_with_summary(
21126            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21127            "",
21128        );
21129        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21130        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21131        state
21132    }
21133
21134    /// A branch rebased onto a main that moved past the recorded
21135    /// `base_commit` is titled from its own first commit, never main's.
21136    #[tokio::test]
21137    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21138        ask_test_home();
21139        let tmp = tempfile::tempdir().unwrap();
21140        let repo = tmp.path().join("repo");
21141        std::fs::create_dir_all(&repo).unwrap();
21142        init_repo(&repo);
21143        let origin = tmp.path().join("origin.git");
21144        let g = |dir: &Path, args: &[&str]| {
21145            let out = std::process::Command::new("git")
21146                .args(args)
21147                .current_dir(dir)
21148                .quiet()
21149                .output()
21150                .expect("spawn git");
21151            assert!(
21152                out.status.success(),
21153                "git {args:?}: {}",
21154                String::from_utf8_lossy(&out.stderr)
21155            );
21156        };
21157        g(
21158            tmp.path(),
21159            &[
21160                "clone",
21161                "--bare",
21162                "-q",
21163                repo.to_str().unwrap(),
21164                origin.to_str().unwrap(),
21165            ],
21166        );
21167        g(
21168            &repo,
21169            &["remote", "add", "origin", origin.to_str().unwrap()],
21170        );
21171        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21172
21173        // Main moves on; the branch is built on top of the new main.
21174        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21175        g(&repo, &["add", "-A"]);
21176        g(
21177            &repo,
21178            &["commit", "-q", "-m", "chore(deps): update a crate"],
21179        );
21180        g(&repo, &["push", "-q", "origin", "main"]);
21181        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21182        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21183        g(&repo, &["add", "-A"]);
21184        g(
21185            &repo,
21186            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21187        );
21188        g(&repo, &["checkout", "-q", "main"]);
21189
21190        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21191        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21192        // Without a readable tracking ref the recorded base's merge base is used.
21193        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21194        assert_eq!(fallback, c1);
21195
21196        let mut state = review_state(&[
21197            "chore(deps): update a crate",
21198            "fix(daemon): apply a chosen action",
21199        ]);
21200        state.repo = repo.clone();
21201        state.base_branch = "main".to_owned();
21202        state.base_commit = c1;
21203        refresh_reviewed_commits(&mut state, "feat/own").await;
21204        assert_eq!(
21205            state.reviewed_commits,
21206            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21207        );
21208        assert_eq!(
21209            review_title(&state).as_deref(),
21210            Some("fix(daemon): apply a chosen action")
21211        );
21212        assert_eq!(
21213            leaked_subjects(&state, "feat/own").await,
21214            Some(vec!["chore(deps): update a crate".to_owned()])
21215        );
21216        // A stale tracking ref is still used when the fetch fails, but the
21217        // leak list is withheld.
21218        state.config.merge.remote = "nowhere".to_owned();
21219        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21220    }
21221
21222    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21223    /// with `origin/main` one commit behind it, and a summarizing agent that
21224    /// prints whatever `reply` holds.
21225    struct SummaryFixture {
21226        _tmp: tempfile::TempDir,
21227        repo: PathBuf,
21228        reply: PathBuf,
21229        runner: Runner,
21230    }
21231
21232    fn summary_fixture() -> SummaryFixture {
21233        ask_test_home();
21234        let tmp = tempfile::tempdir().unwrap();
21235        let repo = tmp.path().join("repo");
21236        std::fs::create_dir_all(&repo).unwrap();
21237        init_repo(&repo);
21238        let origin = tmp.path().join("origin.git");
21239        let g = |args: &[&str]| {
21240            let out = std::process::Command::new("git")
21241                .args(args)
21242                .current_dir(&repo)
21243                .quiet()
21244                .output()
21245                .expect("spawn git");
21246            assert!(
21247                out.status.success(),
21248                "git {args:?}: {}",
21249                String::from_utf8_lossy(&out.stderr)
21250            );
21251        };
21252        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21253        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21254        g(&["push", "-q", "origin", "main"]);
21255        g(&["checkout", "-q", "-b", "feat/own"]);
21256        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21257        g(&["add", "-A"]);
21258        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21259        let reply = tmp.path().join("reply.txt");
21260        std::fs::write(
21261            &reply,
21262            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21263        )
21264        .unwrap();
21265        let mut runner = runner_at(RunStatus::Gating);
21266        let mut state = review_state(&["feat(own): add the own file"]);
21267        state.repo = repo.clone();
21268        state.base_branch = "main".to_owned();
21269        state.candidates[0].branch = "feat/own".to_owned();
21270        state.candidates[0].worktree = repo.clone();
21271        let mut author = spec("alpha");
21272        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21273        author
21274            .env
21275            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21276        state.config.agents = vec![author];
21277        runner.state = state;
21278        SummaryFixture {
21279            _tmp: tmp,
21280            repo,
21281            reply,
21282            runner,
21283        }
21284    }
21285
21286    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21287        let winner = f.runner.state.candidates[0].clone();
21288        f.runner
21289            .guarded_pr_message(&winner, None, true)
21290            .await
21291            .unwrap()
21292            .expect("an English description never parks")
21293    }
21294
21295    fn clean_round_with_minor_findings() -> ReviewRound {
21296        let mut round = passing_round(2, None);
21297        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21298        round.reviews = vec![ReviewRecord {
21299            attempts: 0,
21300            reviewer: 1,
21301            agent: "alpha".into(),
21302            summary: "内部レビュー".into(),
21303            findings: vec![
21304                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21305                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21306            ],
21307            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21308            failed: None,
21309            duration_ms: 0,
21310        }];
21311        round
21312    }
21313
21314    #[tokio::test]
21315    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21316        for review_only in [false, true] {
21317            let mut f = summary_fixture();
21318            if !review_only {
21319                f.runner.state.candidates[0].agent = "alpha".into();
21320            }
21321            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21322            assert!(f.runner.state.reviews[0].clean);
21323            assert!(f.runner.state.reviews[0].fix.is_none());
21324            assert!(f.runner.state.open_findings().is_empty());
21325            assert_eq!(
21326                crate::pr_outcome::required_finding_ids(&f.runner.state),
21327                ["R2-1-1", "R2-1-2"]
21328            );
21329            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21330            let evidence = crate::pr_outcome::summary_evidence(
21331                &f.runner.state,
21332                crate::pr_outcome::FinalHead::At(&head),
21333            );
21334            assert!(
21335                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21336            );
21337            // The ordinary reply omits both IDs: no cached description is adopted.
21338            let omitted = posted(&mut f).await;
21339            assert!(omitted.body.contains("No description"));
21340            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21341            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21342            assert!(f.runner.state.change_summary.is_none());
21343            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21344            let described = posted(&mut f).await;
21345            assert!(described.body.contains("improve documentation for callers"));
21346            assert!(described.body.contains("remaining edge case"));
21347            assert!(described.body.contains("## Open review findings"));
21348            assert!(!described.body.contains("未修正"));
21349        }
21350    }
21351
21352    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21353        let mut f = summary_fixture();
21354        if !review_only {
21355            f.runner.state.candidates[0].agent = "alpha".into();
21356        }
21357        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21358        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21359        fixed
21360            .fix
21361            .as_mut()
21362            .unwrap()
21363            .rejected
21364            .push(crate::verdict::Rejection {
21365                id: "R1-1-1".into(),
21366                why: "呼び出されないコードのため辞退".into(),
21367            });
21368        let mut clean = clean_round_with_minor_findings();
21369        clean.verified_head = Some(head.clone());
21370        f.runner.state.reviews = vec![fixed, clean];
21371        ok_gate(&mut f.runner.state, Some(&head));
21372        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21373        f.runner.state.config.agents.push(scripted("beta", reply));
21374        f.runner.state.config.roles.language_judge =
21375            Some(crate::config::AgentChoice::One("beta".into()));
21376        f
21377    }
21378
21379    #[tokio::test]
21380    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21381        for review_only in [false, true] {
21382            let mut f = rewrite_fixture_with_findings(review_only,
21383                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21384            let m = posted(&mut f).await;
21385            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21386            let expected =
21387                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21388            assert!(m.body.contains(&expected));
21389            assert!(m.body.contains("## Review rounds"));
21390            assert!(
21391                m.body
21392                    .contains("Gate (`verify.gate`): passed on the final commit")
21393            );
21394            assert!(
21395                m.body
21396                    .contains("Tests (`verify.e2e`): passed on the final commit")
21397            );
21398            let c = f.runner.state.change_summary.as_ref().unwrap();
21399            assert!(
21400                m.body
21401                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21402            );
21403            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21404            assert!(crate::pr_outcome::covers_required_findings(
21405                &f.runner.state,
21406                &m.body
21407            ));
21408            assert!(m.body.contains("documentation remains incomplete"));
21409            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21410            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21411            assert!(f.runner.state.github_text.is_none());
21412        }
21413    }
21414
21415    #[tokio::test]
21416    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21417        for review_only in [false, true] {
21418            for reply in [
21419                "TITLE: fix: public description\nA short English description.",
21420                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21421            ] {
21422                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21423                let winner = f.runner.state.candidates[0].clone();
21424                let posted = f
21425                    .runner
21426                    .guarded_pr_message(&winner, None, true)
21427                    .await
21428                    .unwrap();
21429                assert!(posted.is_none());
21430                assert!(f.runner.state.parked);
21431                let withheld = f.runner.state.github_text.as_ref().unwrap();
21432                assert!(
21433                    withheld.rewrite.is_none(),
21434                    "an incomplete rewrite must not be selectable"
21435                );
21436                let q = ask::Questions::open()
21437                    .get(withheld.question.as_ref().unwrap())
21438                    .unwrap();
21439                assert!(
21440                    !q.choices
21441                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21442                );
21443                assert!(
21444                    q.detail.contains("R1-1-1")
21445                        && q.detail.contains("R2-1-1")
21446                        && q.detail.contains("R2-1-2")
21447                );
21448            }
21449        }
21450    }
21451
21452    #[tokio::test]
21453    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21454        for review_only in [false, true] {
21455            let mut f = summary_fixture();
21456            if !review_only {
21457                f.runner.state.candidates[0].agent = "alpha".into();
21458            }
21459            f.runner.state.instruction = "内部タスクの説明".into();
21460            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21461            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21462            f.runner.state.reviews = vec![
21463                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21464                round_with_notes(2, Some("二回目の内部修正報告です。")),
21465                passing_round(3, Some(&head)),
21466            ];
21467            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21468            ok_gate(&mut f.runner.state, Some(&head));
21469            let captured = f.repo.join("public-prompt.txt");
21470            let author = &mut f.runner.state.config.agents[0];
21471            author.command = vec![
21472                "sh".into(),
21473                "-c".into(),
21474                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21475            ];
21476            author.env.insert(
21477                "PROMPT_FILE".into(),
21478                captured.to_string_lossy().into_owned(),
21479            );
21480            let m = posted(&mut f).await;
21481            assert!(m.body.contains("Adds `own.txt`"));
21482            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21483            assert!(m.body.contains("R1-1-1"));
21484            assert!(
21485                m.body
21486                    .contains("Gate (`verify.gate`): passed on the final commit")
21487            );
21488            assert!(
21489                m.body
21490                    .contains("Tests (`verify.e2e`): passed on the final commit")
21491            );
21492            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21493            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21494            let prompt = std::fs::read_to_string(captured).unwrap();
21495            assert!(prompt.contains(&format!(
21496                "git diff {} {head}",
21497                f.runner.state.change_summary.as_ref().unwrap().base
21498            )));
21499            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21500            assert_eq!(
21501                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21502                "二回目の内部修正報告です。"
21503            );
21504        }
21505    }
21506
21507    #[tokio::test]
21508    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21509        for review_only in [false, true] {
21510            let mut f = summary_fixture();
21511            if !review_only {
21512                f.runner.state.candidates[0].agent = "alpha".into();
21513            }
21514            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21515            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21516            std::fs::write(&f.reply, "no usable description").unwrap();
21517            let failed = posted(&mut f).await;
21518            assert!(
21519                failed
21520                    .body
21521                    .contains("No description of what this change does could be generated")
21522            );
21523            assert!(failed.body.is_ascii());
21524            assert!(f.runner.state.github_text.is_none());
21525            assert!(f.runner.state.change_summary.is_none());
21526            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21527            let evidence = crate::pr_outcome::summary_evidence(
21528                &f.runner.state,
21529                crate::pr_outcome::FinalHead::At(&head),
21530            );
21531            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21532            assert!(
21533                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21534                    .exists()
21535            );
21536            f.runner.state =
21537                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21538            std::fs::write(
21539                &f.reply,
21540                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21541            )
21542            .unwrap();
21543            let retried = posted(&mut f).await;
21544            assert!(retried.body.contains("Adds the committed own file."));
21545            assert_eq!(
21546                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21547                "日本語の修正報告"
21548            );
21549        }
21550    }
21551
21552    #[tokio::test]
21553    async fn same_head_verification_and_findings_invalidate_public_prose() {
21554        let mut f = summary_fixture();
21555        posted(&mut f).await;
21556        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21557        let original = f
21558            .runner
21559            .state
21560            .change_summary
21561            .as_ref()
21562            .unwrap()
21563            .evidence
21564            .clone();
21565        ok_gate(&mut f.runner.state, Some(&head));
21566        std::fs::write(
21567            &f.reply,
21568            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21569        )
21570        .unwrap();
21571        let verified = posted(&mut f).await;
21572        assert!(verified.body.contains("now verified"));
21573        assert_ne!(
21574            f.runner.state.change_summary.as_ref().unwrap().evidence,
21575            original
21576        );
21577        let mut round = round_with_notes(1, Some("内部報告"));
21578        round.reviews.push(ReviewRecord {
21579            attempts: 0,
21580            reviewer: 1,
21581            agent: "alpha".into(),
21582            summary: "内部レビュー".into(),
21583            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21584            vote: None,
21585            failed: None,
21586            duration_ms: 0,
21587        });
21588        round
21589            .fix
21590            .as_mut()
21591            .unwrap()
21592            .rejected
21593            .push(crate::verdict::Rejection {
21594                id: "R1-1-1".into(),
21595                why: "呼び出し元から到達できない".into(),
21596            });
21597        f.runner.state.reviews = vec![round];
21598        // A description omitting the required ID is unusable, even with a title.
21599        let omitted = posted(&mut f).await;
21600        assert!(omitted.body.contains("No description"));
21601        assert!(omitted.body.contains("R1-1-1"));
21602        assert!(!omitted.body.contains("到達"));
21603        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21604        let described = posted(&mut f).await;
21605        assert!(described.body.contains("unreachable from callers"));
21606        assert!(described.body.contains("R1-1-1"));
21607        assert!(!described.body.contains("到達"));
21608        let evidence = f
21609            .runner
21610            .state
21611            .change_summary
21612            .as_ref()
21613            .unwrap()
21614            .evidence
21615            .clone();
21616        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21617        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21618        assert!(posted(&mut f).await.body.contains("rationale updated"));
21619        assert_ne!(
21620            f.runner.state.change_summary.as_ref().unwrap().evidence,
21621            evidence
21622        );
21623        // Old serialized summaries have no evidence fingerprint and must regenerate.
21624        f.runner
21625            .state
21626            .change_summary
21627            .as_mut()
21628            .unwrap()
21629            .evidence
21630            .clear();
21631        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21632        assert!(posted(&mut f).await.body.contains("current specification"));
21633    }
21634
21635    #[tokio::test]
21636    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21637        let mut f = summary_fixture();
21638        let m = posted(&mut f).await;
21639        assert_eq!(m.title, "feat(own): add the own file");
21640        assert!(
21641            m.body
21642                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21643            "{}",
21644            m.body
21645        );
21646        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21647        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21648        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21649        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21650        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21651        let kept = f.runner.state.change_summary.as_ref().unwrap();
21652        assert_eq!(
21653            (kept.head.as_str(), kept.base.as_str()),
21654            (head.as_str(), base.as_str())
21655        );
21656        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21657    }
21658
21659    #[tokio::test]
21660    async fn review_only_summary_receives_the_configured_github_language() {
21661        let mut f = summary_fixture();
21662        f.runner.state.config.graph.github_language = "ja".into();
21663        let captured = f.repo.join("summary-prompt.txt");
21664        let author = &mut f.runner.state.config.agents[0];
21665        author.command = vec![
21666            "sh".into(),
21667            "-c".into(),
21668            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21669        ];
21670        author.env.insert(
21671            "PROMPT_FILE".into(),
21672            captured.to_string_lossy().into_owned(),
21673        );
21674        std::fs::write(
21675            &f.reply,
21676            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21677        )
21678        .unwrap();
21679        let m = posted(&mut f).await;
21680        let prompt = std::fs::read_to_string(captured).unwrap();
21681        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21682        assert!(!prompt.contains("in English"));
21683        assert_eq!(m.title, "feat(own): ファイルを追加");
21684        assert!(m.body.contains("own.txt を追加します。"));
21685    }
21686
21687    #[tokio::test]
21688    async fn review_fixes_come_after_the_whole_change_summary() {
21689        let mut f = summary_fixture();
21690        f.runner.state.reviews = vec![
21691            round_with_notes(1, Some("handled the PATH case")),
21692            round_with_notes(2, None),
21693        ];
21694        let m = posted(&mut f).await;
21695        let summary = m.body.find("## Summary").expect("summary");
21696        let fixes = m.body.find("## Review rounds").expect("review rounds");
21697        assert!(summary < fixes, "{}", m.body);
21698        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21699        assert!(m.body.contains("## Final verification"), "{}", m.body);
21700    }
21701
21702    #[tokio::test]
21703    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21704        let mut f = summary_fixture();
21705        let first = posted(&mut f).await;
21706        let first_head = f.runner.state.change_summary.clone().unwrap().head;
21707        // The same head reuses the record: a different reply is not read.
21708        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21709        let again = posted(&mut f).await;
21710        assert_eq!((again.title, again.body), (first.title, first.body));
21711
21712        let g = |args: &[&str]| {
21713            let out = std::process::Command::new("git")
21714                .args(args)
21715                .current_dir(&f.repo)
21716                .quiet()
21717                .output()
21718                .expect("spawn git");
21719            assert!(out.status.success(), "git {args:?}");
21720        };
21721        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21722        g(&["add", "-A"]);
21723        g(&["commit", "-q", "-m", "fix(own): handle more"]);
21724        let second = posted(&mut f).await;
21725        let kept = f.runner.state.change_summary.clone().unwrap();
21726        assert_ne!(kept.head, first_head);
21727        assert!(second.body.contains("Not read yet."), "{}", second.body);
21728        assert!(!second.body.contains("so the first summary is read"));
21729        assert_eq!(second.title, "feat(own): other");
21730    }
21731
21732    #[tokio::test]
21733    async fn an_unavailable_summary_is_stated_and_never_invented() {
21734        let mut f = summary_fixture();
21735        std::fs::write(&f.reply, "no title line here").unwrap();
21736        let m = posted(&mut f).await;
21737        assert!(
21738            m.body
21739                .contains("No description of what this change does could be generated"),
21740            "{}",
21741            m.body
21742        );
21743        assert!(m.body.contains("internal reports remain in local records"));
21744        assert!(f.runner.state.change_summary.is_none());
21745        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21746        // A preview asks nobody and claims nothing.
21747        let winner = f.runner.state.candidates[0].clone();
21748        let preview = f
21749            .runner
21750            .guarded_pr_message(&winner, None, false)
21751            .await
21752            .unwrap()
21753            .unwrap();
21754        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21755    }
21756
21757    #[test]
21758    fn pr_message_review_single_commit_uses_its_subject() {
21759        let state = review_state(&["feat(nats): per-role user"]);
21760        let m = pr_message(&state, 'A');
21761        assert_eq!(m.title, "feat(nats): per-role user");
21762        assert!(!m.body.contains("Review the work already"), "{}", m.body);
21763        assert!(m.body.contains("## Commits under review"), "{}", m.body);
21764    }
21765
21766    #[test]
21767    fn pr_message_review_multi_commit_takes_the_oldest() {
21768        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21769        let m = pr_message(&state, 'A');
21770        assert_eq!(m.title, "feat: the change");
21771        for s in ["feat: the change", "fix: typo", "fix: again"] {
21772            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21773        }
21774    }
21775
21776    #[test]
21777    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21778        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21779            let mut state = review_state(&[first, "fix: later fixup"]);
21780            state.candidates[0].branch = "機能/ログイン".to_owned();
21781            let m = pr_message(&state, 'A');
21782            assert!(
21783                m.title.starts_with("chore: land candidate A of run"),
21784                "{}",
21785                m.title
21786            );
21787        }
21788    }
21789
21790    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21791        BranchFacts {
21792            commits: commits
21793                .iter()
21794                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21795                .collect(),
21796            stat: stat.to_owned(),
21797        }
21798    }
21799
21800    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
21801        let mut r = review_round(true, 0, 1, 1, true, true);
21802        r.round = round;
21803        r.fix = notes.map(|n| FixRecord {
21804            agent: "fixer".to_owned(),
21805            addressed: Vec::new(),
21806            rejected: Vec::new(),
21807            notes: n.to_owned(),
21808            committed: true,
21809            failed: None,
21810            duration_ms: 0,
21811            continuation: None,
21812        });
21813        r
21814    }
21815
21816    #[test]
21817    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
21818        let state = review_state(&["ignored"]);
21819        let f = facts(
21820            &[
21821                (
21822                    "fix(login): resolve PATH on macOS",
21823                    "Login shells skip rc files.",
21824                ),
21825                ("fix: address review", ""),
21826            ],
21827            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
21828        );
21829        let m = pr_message_with(&state, 'A', Some(&f));
21830        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
21831        assert!(
21832            m.body
21833                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
21834            "{}",
21835            m.body
21836        );
21837        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
21838        assert!(m.body.contains("## Diff stat"), "{}", m.body);
21839        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
21840        for banned in [
21841            "Review the work already",
21842            "no task statement",
21843            "Original task",
21844        ] {
21845            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
21846        }
21847        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
21848    }
21849
21850    #[test]
21851    fn pr_message_review_truncates_a_huge_first_commit_body() {
21852        let state = review_state(&["ignored"]);
21853        let f = facts(
21854            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
21855            "s",
21856        );
21857        let m = pr_message_with(&state, 'A', Some(&f));
21858        assert!(m.body.len() < 30_000, "{}", m.body.len());
21859        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
21860        assert!(
21861            m.body.contains("- ... 1 more commit(s)"),
21862            "{}",
21863            m.body.len()
21864        );
21865        assert!(m.body.ends_with("magi:candidate-a\n"));
21866    }
21867
21868    #[test]
21869    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
21870        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
21871        assert_eq!(m.title, "feat: x");
21872        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
21873        assert!(!m.body.contains("Diff stat"), "{}", m.body);
21874        assert!(!m.body.contains("no task statement"), "{}", m.body);
21875    }
21876
21877    #[test]
21878    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
21879        let mut state = review_state(&["日本語の件名"]);
21880        state.candidates[0].branch = "fix/macos-login-path".to_owned();
21881        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
21882        state.candidates[0].branch = "機能/ログイン".to_owned();
21883        assert!(
21884            pr_message(&state, 'A')
21885                .title
21886                .starts_with("chore: land candidate A")
21887        );
21888    }
21889
21890    #[test]
21891    fn pr_message_review_fixes_survive_a_clean_final_round() {
21892        let mut state = review_state(&["feat: x"]);
21893        state.reviews = vec![
21894            round_with_notes(1, Some("handled the PATH case")),
21895            round_with_notes(2, None),
21896        ];
21897        let body = pr_message(&state, 'A').body;
21898        assert!(
21899            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
21900            "{body}"
21901        );
21902        assert!(body.contains("### Round 1"), "{body}");
21903
21904        state.reviews = vec![
21905            round_with_notes(1, Some("first fix")),
21906            round_with_notes(2, Some("")),
21907            round_with_notes(3, Some("second fix")),
21908            round_with_notes(4, None),
21909        ];
21910        let body = pr_message(&state, 'A').body;
21911        assert!(
21912            body.contains("### Round 1") && !body.contains("first fix"),
21913            "{body}"
21914        );
21915        assert!(
21916            body.contains("### Round 3") && !body.contains("second fix"),
21917            "{body}"
21918        );
21919        assert!(body.contains("### Round 2"), "{body}");
21920    }
21921
21922    const FINAL: &str = "abcdef1234567890";
21923
21924    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
21925        let mut r = round_with_notes(round, None);
21926        r.verified_head = verified.map(str::to_owned);
21927        r
21928    }
21929
21930    fn ok_gate(state: &mut RunState, head: Option<&str>) {
21931        state.gate = vec![CommandOutcome {
21932            command: "gate".to_owned(),
21933            code: Some(0),
21934            output_tail: String::new(),
21935            duration_ms: 0,
21936            resource_blocked: false,
21937        }];
21938        state.gate_ran = true;
21939        state.gate_head = head.map(str::to_owned);
21940    }
21941
21942    fn body_at(state: &RunState) -> String {
21943        pr_message_raw(
21944            state,
21945            'A',
21946            None,
21947            crate::pr_outcome::FinalHead::At(FINAL),
21948            ChangeNote::NotAsked,
21949        )
21950        .body
21951    }
21952
21953    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
21954
21955    #[test]
21956    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
21957        let mut state = review_state(&["feat: x"]);
21958        state.reviews = vec![
21959            round_with_notes(1, Some(SANDBOXED)),
21960            passing_round(2, Some(FINAL)),
21961        ];
21962        ok_gate(&mut state, Some(FINAL));
21963        let body = body_at(&state);
21964        assert!(!body.contains("Handled the PATH case."), "{body}");
21965        assert!(!body.contains("could not commit"), "{body}");
21966        assert!(!body.contains("could not run"), "{body}");
21967        assert!(
21968            body.contains("The fix is committed on the branch"),
21969            "{body}"
21970        );
21971        assert!(!body.contains("sandbox"), "{body}");
21972        assert!(
21973            body.contains("Tests (`verify.e2e`): passed on the final commit"),
21974            "{body}"
21975        );
21976        assert!(
21977            body.contains("Gate (`verify.gate`): passed on the final commit"),
21978            "{body}"
21979        );
21980        // The raw report is untouched in the record.
21981        assert!(
21982            state.reviews[0]
21983                .fix
21984                .as_ref()
21985                .unwrap()
21986                .notes
21987                .contains("could not commit")
21988        );
21989    }
21990
21991    #[test]
21992    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
21993        let mut state = review_state(&["feat: x"]);
21994        let mut r1 = round_with_notes(1, Some(SANDBOXED));
21995        r1.fix.as_mut().unwrap().committed = false;
21996        let mut r2 = passing_round(2, Some(FINAL));
21997        r2.e2e[0].code = Some(1);
21998        state.reviews = vec![r1, r2];
21999        let body = body_at(&state);
22000        assert!(!body.contains("could not commit"), "{body}");
22001        assert!(!body.contains("could not run"), "{body}");
22002        assert!(body.contains("This round produced no commit"), "{body}");
22003        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22004        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22005        assert!(!body.contains("passed"), "{body}");
22006    }
22007
22008    #[test]
22009    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22010        let mut state = review_state(&["feat: x"]);
22011        state.reviews = vec![
22012            round_with_notes(1, Some(SANDBOXED)),
22013            passing_round(2, Some("0123456789abcdef")),
22014        ];
22015        ok_gate(&mut state, Some("0123456789abcdef"));
22016        let body = body_at(&state);
22017        assert!(!body.contains("HTTP tests could not run"), "{body}");
22018        // The commit claim is still settled by the structured record.
22019        assert!(!body.contains("could not commit"), "{body}");
22020        assert!(
22021            body.contains("passed on an earlier commit (`0123456`)"),
22022            "{body}"
22023        );
22024        assert!(!body.contains("passed on the final commit"), "{body}");
22025
22026        // A gate recorded before `gate_head` existed is "not recorded".
22027        ok_gate(&mut state, None);
22028        let body = body_at(&state);
22029        assert!(
22030            body.contains("passed on a commit that was not recorded"),
22031            "{body}"
22032        );
22033    }
22034
22035    #[test]
22036    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22037        let state = state_with_summary(
22038            "add retries to the client",
22039            "TITLE: feat: retries\n\nDid it.",
22040        );
22041        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22042        assert_eq!(m.title, "feat: retries");
22043        assert!(
22044            m.body.contains("<summary>Original task</summary>"),
22045            "{}",
22046            m.body
22047        );
22048        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22049        assert!(
22050            m.body
22051                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22052            "{}",
22053            m.body
22054        );
22055    }
22056
22057    #[test]
22058    fn pr_message_review_bounds_a_long_english_subject() {
22059        let long = format!("feat: {}", "word ".repeat(100));
22060        let m = pr_message(&review_state(&[&long]), 'A');
22061        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22062        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22063    }
22064
22065    #[test]
22066    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22067        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22068        let m = pr_message(&state, 'A');
22069        assert_eq!(m.title, "add retries");
22070        assert!(m.body.contains("<summary>Original task</summary>"));
22071        assert!(!m.body.contains("Commits under review"));
22072        assert_eq!(landing_subject_source(&state), state.instruction);
22073    }
22074
22075    #[test]
22076    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22077        let state = review_state(&["feat: the change", "fix: typo"]);
22078        let source = landing_subject_source(&state);
22079        assert_eq!(land::merge_subject("", &source), "feat: the change");
22080        assert_eq!(
22081            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22082            "feat: the change"
22083        );
22084        // An operator's rename still wins.
22085        assert_eq!(
22086            land::merge_subject("feat: renamed by hand", &source),
22087            "feat: renamed by hand"
22088        );
22089        let blank = review_state(&["日本語"]);
22090        assert!(
22091            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22092        );
22093    }
22094
22095    #[test]
22096    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22097        let state = review_state(&["feat: the change"]);
22098        let source = landing_subject_source(&state);
22099        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22100        assert_eq!(
22101            land::merge_subject(landing_title(&state, old), &source),
22102            "feat: the change"
22103        );
22104        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22105        let task = state_with_summary("add retries", "");
22106        assert_eq!(landing_title(&task, old), old);
22107    }
22108
22109    #[test]
22110    fn pr_message_describes_the_change_not_the_task() {
22111        let state = state_with_summary(
22112            "今回やってほしいこと: results projector を直す",
22113            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22114        );
22115        let m = pr_message(&state, 'A');
22116        assert_eq!(m.title, "fix(web): batch the runs list reads");
22117        assert!(
22118            m.body.starts_with("## Summary\n\n- reads run.json once"),
22119            "{}",
22120            m.body
22121        );
22122        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22123        let task_at = m.body.find("今回やってほしいこと").unwrap();
22124        let details_at = m.body.find("<details>").unwrap();
22125        assert!(
22126            details_at < task_at,
22127            "the task lives inside <details>: {}",
22128            m.body
22129        );
22130        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22131        assert!(m.body.contains("magi:candidate-a"));
22132    }
22133
22134    #[test]
22135    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22136        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22137        let m = pr_message(&state, 'A');
22138        assert_eq!(m.title, "add retries");
22139        assert!(
22140            m.body.contains("## Summary\n\n- did some things"),
22141            "{}",
22142            m.body
22143        );
22144
22145        let none = RunState::new(
22146            PathBuf::from("/repo"),
22147            "main".to_owned(),
22148            "abc1234".to_owned(),
22149            "add retries".to_owned(),
22150            Config::default(),
22151        );
22152        let m = pr_message(&none, 'A');
22153        assert_eq!(m.title, "add retries");
22154        assert!(!m.body.contains("## Summary"), "{}", m.body);
22155    }
22156
22157    #[test]
22158    fn pr_message_refuses_the_candidate_commit_subject() {
22159        for bad in [
22160            "TITLE: magi: candidate A (uncommitted work)",
22161            "TITLE: chore: stuff (uncommitted work)",
22162            "TITLE:   ",
22163        ] {
22164            let state = state_with_summary("add retries", bad);
22165            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22166        }
22167    }
22168
22169    #[test]
22170    fn pr_message_bounds_a_very_long_task_and_title() {
22171        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22172        let state = state_with_summary(&long, "- nothing");
22173        let m = pr_message(&state, 'A');
22174        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22175        assert!(!m.title.contains('\n'));
22176
22177        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22178        let m = pr_message(&state, 'A');
22179        assert!(m.title.starts_with("feat: "));
22180        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22181        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22182    }
22183
22184    fn long_title_of(instruction: &str) -> String {
22185        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22186    }
22187
22188    #[test]
22189    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22190        let first = "Make the landing path keep a readable title for long tasks";
22191        let line = format!(
22192            "{first}. {}",
22193            "And then keep going with more words ".repeat(20)
22194        );
22195        let t = long_title_of(&line);
22196        assert_eq!(t, first);
22197        assert!(!t.starts_with("chore: land"));
22198    }
22199
22200    #[test]
22201    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22202        let line = "word ".repeat(200);
22203        let t = long_title_of(&line);
22204        assert!(t.ends_with("word..."), "{t}");
22205        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22206    }
22207
22208    #[test]
22209    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22210        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22211            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22212        }
22213    }
22214
22215    #[test]
22216    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22217        let mut state = state_with_summary("再試行を追加する", "- nothing");
22218        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22219        state.config.graph.github_language = "ja".into();
22220        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22221    }
22222
22223    #[test]
22224    fn pr_message_title_limit_is_exact() {
22225        let at = "a".repeat(PR_TITLE_MAX);
22226        assert_eq!(long_title_of(&at), at);
22227        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22228        assert!(over.ends_with("..."), "{over}");
22229        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22230    }
22231
22232    #[test]
22233    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22234        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22235        let t = long_title_of(&line);
22236        assert!(t.ends_with("..."), "{t}");
22237        assert!(t.is_ascii(), "{t}");
22238    }
22239
22240    #[test]
22241    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22242        let line = format!(
22243            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22244            "plus more filler words ".repeat(20)
22245        );
22246        let t = long_title_of(&line);
22247        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22248    }
22249
22250    #[test]
22251    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22252        let line = format!("\n\n# {}", "title words ".repeat(40));
22253        let state = state_with_summary(&line, "- nothing");
22254        let m = pr_message(&state, 'A');
22255        assert!(m.title.starts_with("title words"), "{}", m.title);
22256        assert_eq!(
22257            land::merge_subject(&m.title, &landing_subject_source(&state)),
22258            m.title
22259        );
22260        // An operator's rename wins untouched.
22261        assert_eq!(
22262            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22263            "feat: renamed by hand"
22264        );
22265    }
22266
22267    #[tokio::test]
22268    async fn github_text_rewrite_is_bounded_and_falls_back() {
22269        crate::run::pin_test_home();
22270        for (reply, accepted) in [
22271            (
22272                "TITLE: fix: retries\nAdd retries for failed requests.",
22273                true,
22274            ),
22275            (
22276                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22277                false,
22278            ),
22279            (
22280                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22281                true,
22282            ),
22283            (
22284                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22285                true,
22286            ),
22287            (
22288                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22289                true,
22290            ),
22291            (
22292                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22293                false,
22294            ),
22295            ("TITLE: fix: retries\n", false),
22296        ] {
22297            let dir = tempfile::tempdir().unwrap();
22298            let mut runner = runner_at(RunStatus::Gating);
22299            runner.state = state_with_summary(
22300                "add retries",
22301                "TITLE: fix: retries\n日本語の説明を書きます。",
22302            );
22303            runner.state.repo = dir.path().to_owned();
22304            runner.state.candidates[0].worktree = dir.path().to_owned();
22305            let mut author = spec("alpha");
22306            author.command = vec![
22307                "sh".into(),
22308                "-c".into(),
22309                "printf '%s' \"$REWRITE_REPLY\"".into(),
22310            ];
22311            author.env.insert("REWRITE_REPLY".into(), reply.into());
22312            runner.state.config.agents = vec![author];
22313            seed_public_summary(&mut runner);
22314            let winner = runner.state.candidates[0].clone();
22315            let first = runner
22316                .guarded_pr_message(&winner, None, true)
22317                .await
22318                .unwrap();
22319            let message = if accepted {
22320                first.expect("a passing rewrite never asks")
22321            } else {
22322                // Withheld: the run parks on a question and posts nothing yet.
22323                assert!(first.is_none());
22324                assert!(runner.state.parked);
22325                let store = ask::Questions::open();
22326                let asked: Vec<_> = store
22327                    .list()
22328                    .into_iter()
22329                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22330                    .collect();
22331                assert_eq!(asked.len(), 1);
22332                assert!(!asked[0].detail.contains("secret"));
22333                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22334                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22335                assert_eq!(
22336                    asked[0]
22337                        .choices
22338                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22339                    rewrite.is_some()
22340                );
22341                // Silence: the question lapses and the neutral text is used,
22342                // without a second rewrite call or a second question.
22343                store
22344                    .update(&asked[0].id, |q| {
22345                        q.abandon("test");
22346                        Ok(())
22347                    })
22348                    .unwrap();
22349                let again = runner
22350                    .guarded_pr_message(&winner, None, true)
22351                    .await
22352                    .unwrap()
22353                    .expect("an abandoned question falls back");
22354                let n = store
22355                    .list()
22356                    .into_iter()
22357                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22358                    .count();
22359                assert_eq!(n, 1, "asked once per rejected text");
22360                again
22361            };
22362            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22363            assert!(!message.body.contains("secret"));
22364            assert!(!message.body.contains("prefix"));
22365            assert!(!message.title.contains("secret"));
22366            assert_eq!(
22367                message.body.contains("Add retries for failed requests."),
22368                accepted
22369            );
22370            assert_eq!(
22371                runner.state.seats["github-text-rewrite"].turns, 1,
22372                "only one rewrite invocation"
22373            );
22374            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22375            if accepted {
22376                assert!(runner.state.github_text.is_none());
22377                assert!(!runner.state.parked);
22378                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22379                assert!(message.body.contains("## Final verification"));
22380                assert!(message.body.contains("Evidence: final diff"));
22381                assert!(
22382                    message
22383                        .body
22384                        .contains(&format!("magi:run/{}", runner.state.id))
22385                );
22386            }
22387            if !accepted {
22388                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22389                assert!(
22390                    message
22391                        .body
22392                        .contains(&format!("magi:run/{}", runner.state.id))
22393                );
22394            }
22395        }
22396    }
22397
22398    fn scripted(id: &str, reply: &str) -> AgentSpec {
22399        let mut a = spec(id);
22400        a.command = vec![
22401            "sh".into(),
22402            "-c".into(),
22403            "printf '%s' \"$REWRITE_REPLY\"".into(),
22404        ];
22405        a.env.insert("REWRITE_REPLY".into(), reply.into());
22406        a
22407    }
22408
22409    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22410    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22411
22412    /// Gate tests inject generated public output, not internal reports.
22413    fn seed_public_summary(runner: &mut Runner) {
22414        let repo = &runner.state.repo;
22415        let g = |args: &[&str]| {
22416            let out = std::process::Command::new("git")
22417                .args(args)
22418                .current_dir(repo)
22419                .quiet()
22420                .output()
22421                .unwrap();
22422            assert!(out.status.success(), "git {args:?}");
22423            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22424        };
22425        g(&["init", "-q", "-b", "main"]);
22426        g(&[
22427            "-c",
22428            "user.name=Test",
22429            "-c",
22430            "user.email=test@example.invalid",
22431            "commit",
22432            "-q",
22433            "--allow-empty",
22434            "-m",
22435            "feat: fixture",
22436        ]);
22437        let head = g(&["rev-parse", "HEAD"]);
22438        g(&["update-ref", "refs/remotes/origin/main", &head]);
22439        let evidence = crate::pr_outcome::summary_evidence(
22440            &runner.state,
22441            crate::pr_outcome::FinalHead::At(&head),
22442        );
22443        let generated = &runner.state.candidates[0].summary;
22444        runner.state.change_summary = Some(crate::run::ChangeSummary {
22445            base: head.clone(),
22446            head,
22447            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22448            title: summary_title(generated).unwrap(),
22449            body: summary_without_title(generated),
22450        });
22451    }
22452
22453    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22454        let mut runner = runner_at(RunStatus::Gating);
22455        runner.state = state_with_summary("add retries", JA_SUMMARY);
22456        if review_only {
22457            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22458        }
22459        runner.state.repo = dir.to_owned();
22460        runner.state.candidates[0].worktree = dir.to_owned();
22461        runner.state.config.agents = agents;
22462        seed_public_summary(&mut runner);
22463        runner
22464    }
22465
22466    fn gate_questions(runner: &Runner) -> usize {
22467        ask::Questions::open()
22468            .list()
22469            .into_iter()
22470            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22471            .count()
22472    }
22473
22474    #[tokio::test]
22475    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22476        crate::run::pin_test_home();
22477        let dir = tempfile::tempdir().unwrap();
22478        let mut runner = rewrite_runner(
22479            dir.path(),
22480            true,
22481            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22482        );
22483        runner.state.config.roles.language_judge =
22484            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22485        let winner = runner.state.candidates[0].clone();
22486        let msg = runner
22487            .guarded_pr_message(&winner, None, true)
22488            .await
22489            .unwrap()
22490            .expect("an English rewrite posts without asking");
22491        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22492        assert!(msg.body.contains("<details>kept</details>"));
22493        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22494        assert_eq!(gate_questions(&runner), 0);
22495        assert!(runner.state.github_text.is_none());
22496        assert!(!runner.state.seats.contains_key("impl-A"));
22497        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22498    }
22499
22500    #[tokio::test]
22501    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22502        crate::run::pin_test_home();
22503        let dir = tempfile::tempdir().unwrap();
22504        let mut runner = rewrite_runner(
22505            dir.path(),
22506            false,
22507            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22508        );
22509        runner.state.config.roles.language_judge =
22510            Some(crate::config::AgentChoice::One("beta".into()));
22511        runner.state.seats.insert(
22512            "impl-A".into(),
22513            SeatState::new("impl-A", "alpha", runner.state.seed),
22514        );
22515        let winner = runner.state.candidates[0].clone();
22516        let msg = runner
22517            .guarded_pr_message(&winner, None, true)
22518            .await
22519            .unwrap()
22520            .expect("rewritten");
22521        assert!(msg.body.contains("<details>kept</details>"));
22522        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22523        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22524        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22525    }
22526
22527    #[tokio::test]
22528    async fn rewrite_reuses_a_resumable_implementer_session() {
22529        crate::run::pin_test_home();
22530        let dir = tempfile::tempdir().unwrap();
22531        let mut runner = rewrite_runner(
22532            dir.path(),
22533            false,
22534            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22535        );
22536        runner.state.config.roles.language_judge =
22537            Some(crate::config::AgentChoice::One("beta".into()));
22538        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22539        seat.turns = 1;
22540        runner.state.seats.insert("impl-A".into(), seat);
22541        let winner = runner.state.candidates[0].clone();
22542        runner
22543            .guarded_pr_message(&winner, None, true)
22544            .await
22545            .unwrap()
22546            .expect("rewritten");
22547        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22548        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22549    }
22550
22551    #[tokio::test]
22552    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22553        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22554        // session, was out of quota, and nothing else was asked.
22555        crate::run::pin_test_home();
22556        let dir = tempfile::tempdir().unwrap();
22557        let mut exhausted = spec("alpha");
22558        exhausted.command = vec!["false".into()];
22559        let mut runner = rewrite_runner(
22560            dir.path(),
22561            false,
22562            vec![exhausted, scripted("beta", EN_REWRITE)],
22563        );
22564        // The implementer's own agent is in the role too: it is not asked twice.
22565        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22566            "alpha".into(),
22567            "beta".into(),
22568        ]));
22569        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22570        seat.turns = 1;
22571        runner.state.seats.insert("impl-A".into(), seat);
22572        let winner = runner.state.candidates[0].clone();
22573        let msg = runner
22574            .guarded_pr_message(&winner, None, true)
22575            .await
22576            .unwrap()
22577            .expect("the role rewrites after the implementer fails");
22578        assert!(msg.body.contains("`src/net.rs`"));
22579        assert_eq!(
22580            runner.state.seats["impl-A"].turns, 2,
22581            "the implementer was tried once"
22582        );
22583        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22584        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22585    }
22586
22587    #[tokio::test]
22588    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22589        crate::run::pin_test_home();
22590        let dir = tempfile::tempdir().unwrap();
22591        let mut broken = spec("bad");
22592        broken.command = vec!["false".into()];
22593        let mut runner = rewrite_runner(
22594            dir.path(),
22595            true,
22596            vec![
22597                broken,
22598                scripted("noformat", "Add retries."),
22599                scripted("good", EN_REWRITE),
22600            ],
22601        );
22602        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22603            "bad".into(),
22604            "noformat".into(),
22605            "good".into(),
22606        ]));
22607        let winner = runner.state.candidates[0].clone();
22608        let msg = runner
22609            .guarded_pr_message(&winner, None, true)
22610            .await
22611            .unwrap()
22612            .expect("the third agent rewrites");
22613        assert!(msg.body.contains("`src/net.rs`"));
22614        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22615        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22616    }
22617
22618    #[tokio::test]
22619    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22620        crate::run::pin_test_home();
22621        // (agents, role): still Japanese; no TITLE; nothing resolvable.
22622        for (agents, role, saved) in [
22623            (
22624                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22625                Some("beta"),
22626                true,
22627            ),
22628            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22629            (Vec::new(), None, false),
22630        ] {
22631            let dir = tempfile::tempdir().unwrap();
22632            let mut runner = rewrite_runner(dir.path(), true, agents);
22633            runner.state.config.roles.language_judge =
22634                role.map(|r| crate::config::AgentChoice::One(r.into()));
22635            let winner = runner.state.candidates[0].clone();
22636            let first = runner
22637                .guarded_pr_message(&winner, None, true)
22638                .await
22639                .unwrap();
22640            assert!(first.is_none(), "withheld text parks the run");
22641            assert!(runner.state.parked);
22642            assert_eq!(gate_questions(&runner), 1);
22643            let g = runner.state.github_text.as_ref().unwrap();
22644            assert_eq!(g.rewrite.is_some(), saved);
22645            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22646        }
22647    }
22648
22649    #[tokio::test]
22650    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22651        crate::run::pin_test_home();
22652        let dir = tempfile::tempdir().unwrap();
22653        let mut runner = runner_at(RunStatus::Gating);
22654        runner.state =
22655            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22656        runner.state.repo = dir.path().to_owned();
22657        runner.state.candidates[0].worktree = dir.path().to_owned();
22658        let mut author = spec("alpha");
22659        author.command = vec![
22660            "sh".into(),
22661            "-c".into(),
22662            "printf '%s' \"$REWRITE_REPLY\"".into(),
22663        ];
22664        author.env.insert(
22665            "REWRITE_REPLY".into(),
22666            format!(
22667                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22668                "English sentence. ".repeat(600)
22669            ),
22670        );
22671        runner.state.config.agents = vec![author];
22672        seed_public_summary(&mut runner);
22673        let winner = runner.state.candidates[0].clone();
22674        assert!(
22675            runner
22676                .guarded_pr_message(&winner, None, true)
22677                .await
22678                .unwrap()
22679                .is_none()
22680        );
22681        let record = runner.state.github_text.clone().unwrap();
22682        let rewrite = record.rewrite.clone().unwrap();
22683        assert_eq!(rewrite.failed, ["body-language"]);
22684        assert!(
22685            rewrite
22686                .body
22687                .ends_with(&format!("magi:run/{}", runner.state.id))
22688        );
22689        let store = ask::Questions::open();
22690        let q = store.get(record.question.as_ref().unwrap()).unwrap();
22691        assert!(
22692            q.choices
22693                .contains(&crate::github_text::USE_REWRITE.to_owned())
22694        );
22695        for label in [
22696            "ORIGINAL",
22697            "FALLBACK",
22698            "REWRITE",
22699            "Rules still failed: body-language",
22700            "truncated",
22701        ] {
22702            assert!(q.detail.contains(label), "{label}");
22703        }
22704        let artifact = std::fs::read_to_string(crate::run::artifact_path(
22705            &runner.state,
22706            &crate::github_text::artifact_name(&record.fingerprint),
22707        ))
22708        .unwrap();
22709        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22710        // Restart from serialized state, then retry after a bad owner title.
22711        runner.state =
22712            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22713        store
22714            .update(&q.id, |q| {
22715                q.say("まだ日本語")?;
22716                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22717            })
22718            .unwrap();
22719        assert!(
22720            runner
22721                .guarded_pr_message(&winner, None, true)
22722                .await
22723                .unwrap()
22724                .is_none()
22725        );
22726        let retry = runner.state.github_text.as_ref().unwrap();
22727        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22728        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22729        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22730        assert!(
22731            q.posting_comparison
22732                .as_ref()
22733                .unwrap()
22734                .matches_detail(&q.detail)
22735        );
22736        store
22737            .update(&q.id, |q| {
22738                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22739            })
22740            .unwrap();
22741        let message = runner
22742            .guarded_pr_message(&winner, None, true)
22743            .await
22744            .unwrap()
22745            .unwrap();
22746        assert_eq!(message.title, rewrite.title);
22747        assert_eq!(message.body, rewrite.body);
22748        assert_eq!(
22749            runner.state.github_text.as_ref().unwrap().choice,
22750            Some(GithubTextChoice::Rewrite)
22751        );
22752        let again = runner
22753            .guarded_pr_message(&winner, None, true)
22754            .await
22755            .unwrap()
22756            .unwrap();
22757        assert_eq!(again.title, message.title);
22758        assert_eq!(again.body, message.body);
22759        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22760        assert!(
22761            !runner
22762                .state
22763                .events
22764                .iter()
22765                .any(|e| e.message.contains(&rewrite.title)
22766                    || e.message.contains("English sentence."))
22767        );
22768    }
22769
22770    #[tokio::test]
22771    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22772        crate::run::pin_test_home();
22773        let dir = tempfile::tempdir().unwrap();
22774        let mut runner = runner_at(RunStatus::Gating);
22775        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22776        runner.state.repo = dir.path().to_owned();
22777        runner.state.candidates[0].worktree = dir.path().to_owned();
22778        seed_public_summary(&mut runner);
22779        let winner = runner.state.candidates[0].clone();
22780        assert!(
22781            runner
22782                .guarded_pr_message(&winner, None, true)
22783                .await
22784                .unwrap()
22785                .is_none()
22786        );
22787        let record = runner.state.github_text.as_ref().unwrap();
22788        assert!(record.rewrite.is_none());
22789        // Records written before the additive field deserialize with no rewrite.
22790        let mut old = serde_json::to_value(record).unwrap();
22791        old.as_object_mut().unwrap().remove("rewrite");
22792        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22793        assert!(old.rewrite.is_none());
22794        let id = record.question.clone().unwrap();
22795        ask::Questions::open()
22796            .update(&id, |q| {
22797                q.choices.push(crate::github_text::USE_REWRITE.into());
22798                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22799            })
22800            .unwrap();
22801        let message = runner
22802            .guarded_pr_message(&winner, None, true)
22803            .await
22804            .unwrap()
22805            .unwrap();
22806        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22807        assert_eq!(
22808            runner.state.github_text.as_ref().unwrap().choice,
22809            Some(GithubTextChoice::Fallback)
22810        );
22811    }
22812
22813    #[tokio::test]
22814    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
22815        crate::run::pin_test_home();
22816        let dir = tempfile::tempdir().unwrap();
22817        let mut runner = runner_at(RunStatus::Gating);
22818        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
22819        runner.state.repo = dir.path().to_owned();
22820        runner.state.candidates[0].worktree = dir.path().to_owned();
22821        seed_public_summary(&mut runner);
22822        let winner = runner.state.candidates[0].clone();
22823        assert!(
22824            runner
22825                .guarded_pr_message(&winner, None, true)
22826                .await
22827                .unwrap()
22828                .is_none()
22829        );
22830        let store = ask::Questions::open();
22831        let find = |run: &str| {
22832            store
22833                .list()
22834                .into_iter()
22835                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
22836                .unwrap()
22837        };
22838        let run = runner.state.id.clone();
22839        let q = find(&run);
22840        let fp = runner
22841            .state
22842            .github_text
22843            .as_ref()
22844            .unwrap()
22845            .fingerprint
22846            .clone();
22847        let name = crate::github_text::artifact_name(&fp);
22848        let path = crate::run::artifact_path(&runner.state, &name);
22849        assert!(q.detail.contains(&path.display().to_string()));
22850        assert!(q.detail.contains("title-language: title"));
22851        let saved = std::fs::read_to_string(&path).expect("artifact written");
22852        assert!(saved.contains("日本語のタイトル"));
22853        assert!(q.cwd.is_none());
22854        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
22855        // A bad replacement is refused and asked about once more.
22856        let say = |id: &str, text: &str, choice: &str| {
22857            store
22858                .update(id, |q| {
22859                    q.thread.push(ask::Turn {
22860                        who: ask::Who::Operator,
22861                        body: text.to_owned(),
22862                        at: jiff::Timestamp::now(),
22863                        note: None,
22864                    });
22865                    q.answer(ask::Answer::Choice(choice.to_owned()))
22866                })
22867                .unwrap();
22868        };
22869        say(&q.id, "まだ日本語", "use my text");
22870        assert!(
22871            runner
22872                .guarded_pr_message(&winner, None, true)
22873                .await
22874                .unwrap()
22875                .is_none()
22876        );
22877        let second = find(&run);
22878        assert_ne!(second.id, q.id);
22879        assert!(second.detail.contains("did not pass"));
22880        // The retry still compares the original text; the rejected
22881        // replacement is named by category only.
22882        assert!(second.detail.contains("日本語のタイトル"));
22883        assert!(!second.detail.contains("まだ日本語"));
22884        assert!(second.detail.contains(&path.display().to_string()));
22885        say(&second.id, "fix: retry failed requests", "use my text");
22886        let message = runner
22887            .guarded_pr_message(&winner, None, true)
22888            .await
22889            .unwrap()
22890            .expect("a vetted replacement is posted");
22891        assert_eq!(message.title, "fix: retry failed requests");
22892        assert_eq!(
22893            runner
22894                .state
22895                .github_text
22896                .as_ref()
22897                .unwrap()
22898                .chosen_title
22899                .as_deref(),
22900            Some("fix: retry failed requests")
22901        );
22902        assert!(runner.state.github_text.as_ref().unwrap().resolved);
22903        // Resumed again: the saved decision is applied, nothing is re-asked.
22904        let again = runner
22905            .guarded_pr_message(&winner, None, true)
22906            .await
22907            .unwrap()
22908            .unwrap();
22909        assert_eq!(again.title, message.title);
22910        assert_eq!(
22911            store
22912                .list()
22913                .into_iter()
22914                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
22915                .count(),
22916            2
22917        );
22918    }
22919
22920    #[test]
22921    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
22922        // What magi itself writes stays English under any configured language,
22923        // so a future localisation of these headings fails here. (The agents'
22924        // own text is also checked by the posting gate.)
22925        let mut state = state_with_summary(
22926            "add retries",
22927            "TITLE: fix(web): batch reads\n- reads run.json once",
22928        );
22929        state.config.graph.language = "ja".to_owned();
22930        let m = pr_message(&state, 'A');
22931        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
22932        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
22933
22934        // The task is the operator's own text: it goes in untouched, and the
22935        // fallback title (no summary) may be in its language too.
22936        let task = "今回やってほしいこと: results projector を直す";
22937        let mut state = state_with_summary(task, "- no title line");
22938        state.config.graph.language = "ja".to_owned();
22939        let m = pr_message(&state, 'A');
22940        assert_eq!(
22941            m.title,
22942            format!("chore: land candidate A of run {}", state.id)
22943        );
22944        assert!(
22945            m.body.contains(&format!(
22946                "<summary>Original task</summary>\n\n{task}\n\n</details>"
22947            )),
22948            "{}",
22949            m.body
22950        );
22951    }
22952
22953    #[test]
22954    fn pr_message_scrubs_home_paths_and_addresses() {
22955        let state = state_with_summary(
22956            "fix it in /Users/someone/src/x",
22957            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
22958        );
22959        let m = pr_message(&state, 'A');
22960        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
22961            assert!(!m.body.contains(leak), "{}", m.body);
22962        }
22963        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
22964    }
22965
22966    #[test]
22967    fn pr_message_survives_a_task_that_closes_details() {
22968        let state = state_with_summary("a </details> b", "TITLE: fix: x");
22969        let m = pr_message(&state, 'A');
22970        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
22971    }
22972
22973    #[test]
22974    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
22975        let cmd = manual_merge_command(
22976            MergeStyle::Squash,
22977            Path::new("/repo"),
22978            "b",
22979            "fix: \"quoted\" $(x) `y`\n\nbody",
22980        );
22981        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
22982    }
22983
22984    #[test]
22985    fn manual_merge_command_matches_the_configured_style() {
22986        let repo = Path::new("/repo");
22987        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
22988
22989        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
22990        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
22991
22992        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
22993        assert_eq!(
22994            squash,
22995            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
22996             \"Merge magi run 0832 (candidate A)\""
22997        );
22998
22999        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23000        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23001    }
23002
23003    #[test]
23004    fn a_nudge_gets_a_quarter_of_the_budget() {
23005        // The judge and implement budgets magi ships with.
23006        assert_eq!(retry_budget(secs(1200), true), secs(300));
23007        assert_eq!(retry_budget(secs(3600), true), secs(900));
23008    }
23009
23010    #[test]
23011    fn a_resent_prompt_keeps_the_whole_budget() {
23012        // The seat kept no context, so the retry is the original job again and
23013        // shortening it would only guarantee a second failure.
23014        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23015        assert_eq!(retry_budget(secs(60), false), secs(60));
23016    }
23017
23018    #[test]
23019    fn the_floor_never_exceeds_the_original_budget() {
23020        // A short configured timeout must not be *raised* by the floor: the
23021        // operator asked for a bound, and a retry may not outlast the attempt
23022        // it is retrying.
23023        assert_eq!(retry_budget(secs(60), true), secs(60));
23024        assert_eq!(retry_budget(secs(480), true), secs(120));
23025        assert_eq!(retry_budget(secs(0), true), secs(0));
23026    }
23027
23028    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23029        agent::CommandEvidence {
23030            id: "item1".to_owned(),
23031            description: "cargo test".to_owned(),
23032            exit_code,
23033            result_summary: String::new(),
23034            source: "codex".to_owned(),
23035        }
23036    }
23037
23038    #[test]
23039    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23040        // No evidence is not the same fact as unconfirmed evidence: a
23041        // backend with no adapter, or a reply that ran no commands at all,
23042        // must not be misread as carrying a dangling job.
23043        assert!(!has_unconfirmed_command(&[]));
23044    }
23045
23046    #[test]
23047    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23048        // Deliberately not a check on the exit code's *value*: a fixer
23049        // legitimately runs something that fails mid-iteration before it
23050        // succeeds, and that must never by itself reopen a valid report.
23051        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23052        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23053        assert!(!has_unconfirmed_command(&[
23054            evidence(Some(0)),
23055            evidence(Some(101))
23056        ]));
23057    }
23058
23059    #[test]
23060    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23061        assert!(has_unconfirmed_command(&[
23062            evidence(Some(0)),
23063            evidence(None)
23064        ]));
23065    }
23066
23067    #[test]
23068    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23069        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23070        assert_eq!(
23071            verified_noop_claim(true, &[], text).as_deref(),
23072            Some("already fixed by b32cfc4, on main.")
23073        );
23074    }
23075
23076    #[test]
23077    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23078        // A timeout or a bad exit code reads as the ordinary loss it is,
23079        // whatever the reply's own prose claims.
23080        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23081        assert!(verified_noop_claim(false, &[], text).is_none());
23082    }
23083
23084    #[test]
23085    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23086        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23087        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23088        // A confirmed command alongside the marker is fine.
23089        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23090    }
23091
23092    #[test]
23093    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23094        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23095    }
23096
23097    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23098    /// pair, in order, labelled A, B, C, ...
23099    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23100        runner.state.candidates = shape
23101            .iter()
23102            .enumerate()
23103            .map(|(i, &(empty, verified))| Candidate {
23104                index: i,
23105                label: (b'A' + i as u8) as char,
23106                agent: "sonnet".to_owned(),
23107                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23108                worktree: PathBuf::from(format!("/wt/{i}")),
23109                summary: String::new(),
23110                stat: String::new(),
23111                files: 0,
23112                commits: 0,
23113                empty,
23114                failed: None,
23115                verified_noop: verified.map(str::to_owned),
23116                duration_ms: 0,
23117                folded: false,
23118            })
23119            .collect();
23120    }
23121
23122    #[test]
23123    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23124        ask_test_home();
23125        let mut runner = runner_at(RunStatus::Implementing);
23126        set_candidates(
23127            &mut runner,
23128            &[
23129                (true, Some("already on main at b32cfc4")),
23130                (true, Some("same fix, see the existing test")),
23131            ],
23132        );
23133
23134        runner
23135            .after_implement()
23136            .expect("a verified no-op is not an error");
23137
23138        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23139    }
23140
23141    #[test]
23142    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23143        ask_test_home();
23144        let mut runner = runner_at(RunStatus::Implementing);
23145        // Candidate A declares a verified no-op; candidate B simply wrote
23146        // nothing and said nothing about why. One candidate's claim is not
23147        // the whole run's agreement.
23148        set_candidates(
23149            &mut runner,
23150            &[(true, Some("already on main at b32cfc4")), (true, None)],
23151        );
23152
23153        let err = runner
23154            .after_implement()
23155            .expect_err("an unverified empty candidate must still fail the run");
23156
23157        assert!(
23158            err.to_string().contains("no candidate produced a change"),
23159            "{err}"
23160        );
23161        assert_eq!(runner.state.status, RunStatus::Failed);
23162    }
23163
23164    #[test]
23165    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23166        ask_test_home();
23167        let mut runner = runner_at(RunStatus::Implementing);
23168        set_candidates(&mut runner, &[(true, None), (true, None)]);
23169
23170        let err = runner
23171            .after_implement()
23172            .expect_err("no candidate declared anything; this is an ordinary failure");
23173
23174        assert!(
23175            err.to_string().contains("no candidate produced a change"),
23176            "{err}"
23177        );
23178        assert_eq!(runner.state.status, RunStatus::Failed);
23179    }
23180}
23181
23182/// A graft rollback whose restore could not be confirmed (see
23183/// [`Runner::graft`]).
23184#[derive(Debug)]
23185struct GraftRestoreFailed(String);
23186
23187impl std::fmt::Display for GraftRestoreFailed {
23188    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23189        write!(
23190            f,
23191            "could not restore the winner to its pre-graft commit: {}",
23192            self.0
23193        )
23194    }
23195}
23196
23197impl std::error::Error for GraftRestoreFailed {}
23198
23199/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23200/// not the agent: they never draw on an agent's own timeout, and running out of
23201/// them never reads as an agent failure.
23202#[cfg(test)]
23203mod resource_wait_tests {
23204    use super::*;
23205    use std::collections::BTreeMap;
23206
23207    fn sh(id: &str, script: &str) -> AgentSpec {
23208        AgentSpec {
23209            id: id.to_owned(),
23210            kind: crate::config::AgentKind::Command,
23211            model: None,
23212            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23213            extra_args: Vec::new(),
23214            env: BTreeMap::new(),
23215            prompt_delivery: None,
23216            max_turn_secs: None,
23217        }
23218    }
23219
23220    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23221        SeatJob {
23222            seat: SeatState::new("impl-A", &spec.id, 1),
23223            spec,
23224            cwd: dir.to_path_buf(),
23225            prompt: "do it".to_owned(),
23226            timeout,
23227            allow_write: true,
23228            sessions: false,
23229            artifacts: dir.join("art"),
23230            stem: "impl-A".to_owned(),
23231            handover: None,
23232        }
23233    }
23234
23235    fn state(resource_wait_secs: u64) -> RunState {
23236        let mut config = Config::default();
23237        config.graph.timeout_resource_wait = resource_wait_secs;
23238        RunState::new(
23239            PathBuf::from("/repo"),
23240            "main".to_owned(),
23241            "abc1234".to_owned(),
23242            "task".to_owned(),
23243            config,
23244        )
23245    }
23246
23247    fn hold(cache: &Path) -> crate::cache::Guard {
23248        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23249        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23250            crate::cache::AcquireOutcome::Acquired(g) => g,
23251            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23252        }
23253    }
23254
23255    async fn run(
23256        jobs: Vec<SeatJob>,
23257        sem: Arc<Semaphore>,
23258        cache: Option<&Path>,
23259        st: &mut RunState,
23260    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23261        let prompts = Prompts::default();
23262        let ctx = WaveCtx {
23263            run: "r",
23264            node: "implement",
23265            prompts: &prompts,
23266            cache,
23267            round: None,
23268            carry_seats: false,
23269            pauses: None,
23270        };
23271        wave(jobs, sem, &ctx, st, 0).await
23272    }
23273
23274    #[tokio::test]
23275    async fn verification_cache_wait_preserves_the_command_budget() {
23276        crate::run::pin_test_home();
23277        let tmp = tempfile::tempdir().unwrap();
23278        let cache = tmp.path().join("target");
23279        let guard = hold(&cache);
23280        let release = tokio::spawn(async move {
23281            tokio::time::sleep(Duration::from_millis(100)).await;
23282            guard.release();
23283        });
23284        let mut st = state(10);
23285        let budget = Duration::from_millis(50);
23286        let wait = repository_wait(&st, None);
23287        let (outcomes, _) = with_cache_lease(
23288            &mut st,
23289            Some(&cache),
23290            "gate",
23291            "gate",
23292            tmp.path(),
23293            "abc1234",
23294            budget,
23295            "test gate",
23296            &wait,
23297            |_, available| async move {
23298                assert_eq!(
23299                    available, budget,
23300                    "contention must not shorten verification"
23301                );
23302                (Vec::new(), false, Vec::new())
23303            },
23304        )
23305        .await
23306        .unwrap();
23307        release.await.unwrap();
23308        assert!(outcomes.is_empty(), "{outcomes:?}");
23309        assert!(
23310            st.events
23311                .iter()
23312                .any(|event| event.message.contains("waiting for"))
23313        );
23314        assert!(st.resource_wait.is_none());
23315        // The verification body no longer owns the lease after returning.
23316        hold(&cache).release();
23317    }
23318
23319    #[tokio::test]
23320    async fn verification_cache_wait_expires_without_starting_commands() {
23321        crate::run::pin_test_home();
23322        let tmp = tempfile::tempdir().unwrap();
23323        let cache = tmp.path().join("target");
23324        let _held = hold(&cache);
23325        let mut st = state(0);
23326        let wait = repository_wait(&st, None);
23327        let (outcomes, _) = with_cache_lease(
23328            &mut st,
23329            Some(&cache),
23330            "gate",
23331            "gate",
23332            tmp.path(),
23333            "abc1234",
23334            Duration::from_secs(60),
23335            "test gate",
23336            &wait,
23337            |_, _| async { panic!("commands must not start without the cache lease") },
23338        )
23339        .await
23340        .unwrap();
23341        assert_eq!(outcomes.len(), 1);
23342        assert!(outcomes[0].resource_blocked);
23343        assert!(st.resource_wait.is_none());
23344        assert!(matches!(
23345            st.resource_block.unwrap().cause,
23346            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23347        ));
23348    }
23349
23350    #[tokio::test]
23351    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23352        crate::run::pin_test_home();
23353        let tmp = tempfile::tempdir().unwrap();
23354        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23355        let cache = tmp.path().join("cache");
23356        let mut state = state(21600);
23357        let wait = repository_wait(&state, None);
23358        let (outcomes, _) = with_cache_lease(
23359            &mut state,
23360            Some(&cache),
23361            "gate",
23362            "gate",
23363            tmp.path(),
23364            "head",
23365            Duration::from_secs(60),
23366            "gate",
23367            &wait,
23368            |_, _| async { panic!("freshness failure must not start verification") },
23369        )
23370        .await
23371        .unwrap();
23372        assert_eq!(outcomes.len(), 1);
23373        assert!(outcomes[0].resource_blocked);
23374        let block = state.resource_block.unwrap();
23375        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23376        assert!(!block.description().contains("timed out"));
23377    }
23378
23379    #[tokio::test]
23380    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23381        crate::run::pin_test_home();
23382        let tmp = tempfile::tempdir().unwrap();
23383        let cache = tmp.path().join("cache");
23384        let _held = hold(&cache);
23385        let mut state = state(21600);
23386        let pause = Pause::new();
23387        let interrupt = Pause::new();
23388        let cause = ParkCause::Operator;
23389        pause.park_for(cause.clone());
23390        let prompts = Prompts::default();
23391        let ctx = WaveCtx {
23392            run: "test",
23393            node: "fix",
23394            prompts: &prompts,
23395            cache: Some(&cache),
23396            round: None,
23397            carry_seats: false,
23398            pauses: Some((&pause, &interrupt)),
23399        };
23400        let result = wave(
23401            vec![job(
23402                sh("writer", "exit 99"),
23403                tmp.path(),
23404                Duration::from_secs(1),
23405            )],
23406            Arc::new(Semaphore::new(1)),
23407            &ctx,
23408            &mut state,
23409            0,
23410        )
23411        .await;
23412        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23413        assert_eq!(result[0].1.turns, 0);
23414        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23415        assert_eq!(
23416            state.resource_block.unwrap().cause,
23417            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23418        );
23419        assert!(state.active.is_empty());
23420    }
23421
23422    #[tokio::test]
23423    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23424        crate::run::pin_test_home();
23425        let tmp = tempfile::tempdir().unwrap();
23426        let cache = tmp.path().join("target-a");
23427        let guard = hold(&cache);
23428        // Held for longer than the agent's whole 1s budget.
23429        tokio::spawn(async move {
23430            tokio::time::sleep(Duration::from_millis(1500)).await;
23431            guard.release();
23432        });
23433        let mut st = state(60);
23434        let results = run(
23435            vec![job(
23436                sh("alpha", "echo ok"),
23437                tmp.path(),
23438                Duration::from_secs(1),
23439            )],
23440            Arc::new(Semaphore::new(1)),
23441            Some(&cache),
23442            &mut st,
23443        )
23444        .await;
23445        assert!(
23446            matches!(results[0].2, AgentOutcome::Ok(_)),
23447            "the wait must not be charged to the agent"
23448        );
23449        assert!(st.resource_block.is_none());
23450        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23451        assert!(st.resource_wait.is_none() && st.active.is_empty());
23452    }
23453
23454    #[tokio::test]
23455    #[cfg(unix)]
23456    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23457        use std::os::fd::AsRawFd as _;
23458        crate::run::pin_test_home();
23459        let tmp = tempfile::tempdir().unwrap();
23460        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23461        let common = tmp.path().join(".git");
23462        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23463        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23464        assert_eq!(
23465            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23466            0
23467        );
23468        let mut st = state(1);
23469        st.config.coordination = crate::config::test_coordination();
23470        let mut reader = job(
23471            sh("reader", "echo must-not-run"),
23472            tmp.path(),
23473            Duration::from_secs(30),
23474        );
23475        reader.allow_write = false;
23476        let results = run(
23477            vec![reader.clone()],
23478            Arc::new(Semaphore::new(1)),
23479            None,
23480            &mut st,
23481        )
23482        .await;
23483        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23484        assert_eq!(results[0].1.turns, 0);
23485        assert!(FailClass::of(&results[0].2).is_none());
23486        assert_eq!(
23487            st.resource_block.as_ref().unwrap().kind,
23488            "repository coordination"
23489        );
23490        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23491        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23492        drop(external);
23493        let editor = crate::repo_lock::SeatLease::for_checkout(
23494            tmp.path(),
23495            true,
23496            crate::config::test_coordination().as_ref(),
23497        )
23498        .await
23499        .unwrap();
23500        let pause = Pause::new();
23501        let interrupt = Pause::new();
23502        let prompts = Prompts::default();
23503        let ctx = WaveCtx {
23504            run: "r",
23505            node: "review",
23506            prompts: &prompts,
23507            cache: None,
23508            round: None,
23509            carry_seats: false,
23510            pauses: Some((&pause, &interrupt)),
23511        };
23512        let (results, ()) = tokio::join!(
23513            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23514            async {
23515                tokio::time::sleep(Duration::from_millis(50)).await;
23516                pause.park();
23517            }
23518        );
23519        assert!(
23520            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23521        );
23522        assert_eq!(results[0].1.turns, 0);
23523        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23524        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23525        drop(editor);
23526    }
23527
23528    #[tokio::test]
23529    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23530        crate::run::pin_test_home();
23531        let tmp = tempfile::tempdir().unwrap();
23532        let cache = tmp.path().join("target-b");
23533        let _held = hold(&cache);
23534        let mut st = state(1);
23535        let results = run(
23536            vec![job(
23537                sh("alpha", "echo ok"),
23538                tmp.path(),
23539                Duration::from_secs(1),
23540            )],
23541            Arc::new(Semaphore::new(1)),
23542            Some(&cache),
23543            &mut st,
23544        )
23545        .await;
23546        let (_, seat, out) = &results[0];
23547        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23548        assert!(wave_blocked(&results));
23549        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23550        assert_eq!(seat.turns, 0, "nothing was asked");
23551        assert!(
23552            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23553            "no agent was spawned"
23554        );
23555        assert!(FailClass::of(out).is_none(), "not a failure class");
23556        let block = st.resource_block.as_ref().expect("recorded for the resume");
23557        assert_eq!(block.kind, "shared build cache");
23558        assert!(matches!(
23559            block.cause,
23560            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23561        ));
23562        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23563        assert!(st.active.is_empty());
23564    }
23565
23566    #[tokio::test]
23567    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23568        crate::run::pin_test_home();
23569        let tmp = tempfile::tempdir().unwrap();
23570        let mut st = state(1);
23571        let results = run(
23572            vec![job(
23573                sh("alpha", "echo ok"),
23574                tmp.path(),
23575                Duration::from_secs(1),
23576            )],
23577            Arc::new(Semaphore::new(0)),
23578            None,
23579            &mut st,
23580        )
23581        .await;
23582        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23583        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23584        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23585    }
23586
23587    #[tokio::test]
23588    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23589        crate::run::pin_test_home();
23590        let tmp = tempfile::tempdir().unwrap();
23591        let mut st = state(60);
23592        let results = run(
23593            vec![job(
23594                sh("alpha", "sleep 30"),
23595                tmp.path(),
23596                Duration::from_secs(1),
23597            )],
23598            Arc::new(Semaphore::new(1)),
23599            None,
23600            &mut st,
23601        )
23602        .await;
23603        let out = &results[0].2;
23604        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23605        let class = FailClass::of(out).expect("a failure");
23606        assert_eq!(class, FailClass::Timeout);
23607        assert!(should_hand_over(None, &class), "existing handover policy");
23608        assert!(st.resource_block.is_none());
23609    }
23610}