Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Recorded before the first save, so a crash right after minting
773        // cannot leave a run with no origin. Legibility only: nothing reads it
774        // to decide anything.
775        state.fetched_base = Some(fetched_base);
776        state.origin = Some(origin);
777        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778        state.status = RunStatus::Prep;
779        state.event("start", format!("run {} created", state.id));
780        startup.finish(&mut state, Ok(()))?;
781        Ok(Self {
782            state,
783            roles,
784            sem: Arc::new(Semaphore::new(max_parallel)),
785            pause: Pause::new(),
786            interrupt: Pause::new(),
787        })
788    }
789
790    /// Open a review-only run against work that already exists on `branch`.
791    ///
792    /// The expensive half of the graph is the implement wave — measured at
793    /// 111 and 134 internal tool-loop turns on this repository, against a
794    /// handful for a judge or a reviewer. The cheap half is worth running on
795    /// hand-written work too, and there was no way to reach it.
796    ///
797    /// No new state and no schema change are needed: a run with **one** viable
798    /// candidate and a tally already decided degrades `execute` to exactly
799    /// review → gate → merge, because `judge` skips a single-candidate field,
800    /// `deliberate` has fewer than two first choices to reconcile, `vote`
801    /// returns early, `tally` is already present and `fold_losers` has no
802    /// losers. Resuming such a run therefore does the right thing as well.
803    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804        Self::review_taking_over(repo, branch, config, None, origin).await
805    }
806
807    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
808    /// at the same task still has `branch` checked out, its worktree is
809    /// released first if that is safe (see [`crate::handover`]), and the
810    /// review refuses with the reason if it is not. `None` is a hand-run
811    /// review: it has no earlier attempts, so only a worktree of a dead run
812    /// magi recorded itself can be released.
813    pub async fn review_taking_over(
814        repo: &Path,
815        branch: &str,
816        config: Config,
817        takeover: Option<crate::handover::Takeover>,
818        origin: Origin,
819    ) -> Result<Self> {
820        let repo = git::toplevel(repo).await?;
821        let missing = agent::missing_programs(&config.agents);
822        if !missing.is_empty() {
823            bail!(
824                "these agent programs are not on PATH: {}. Fix the roster in \
825                 magi.toml or install them.",
826                missing.join(", ")
827            );
828        }
829        let base_branch =
830            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831                .await?;
832        if base_branch == branch {
833            bail!("`{branch}` is the base branch; there is nothing to review against");
834        }
835        let fetched_base = resolve_base(
836            &repo,
837            &base_branch,
838            &config.merge.remote,
839            config.coordination.as_ref(),
840        )
841        .await?;
842        let base_commit = fetched_base.commit.clone();
843
844        let roles = config.resolve_roles()?;
845        let max_parallel = config.graph.max_parallel.max(1);
846        let mut state = RunState::new(
847            repo.clone(),
848            base_branch,
849            base_commit.clone(),
850            format!("Review the work already on branch `{branch}`."),
851            config,
852        );
853        state.fetched_base = Some(fetched_base);
854        state.origin = Some(origin);
855        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857        let opened = async {
858            startup.stage(&mut state, "handover")?;
859
860            // Released before anything else touches the branch: a stale local
861            // branch is moved with `git branch -f`, which git refuses while an
862            // earlier attempt's worktree still has it checked out. Everything
863            // after this point that can fail puts the old run back.
864            // A hand-run review has no task, hence no earlier attempts, but a
865            // worktree of a dead run magi made may still be released.
866            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867                earlier: Vec::new(),
868                home: crate::run::home(),
869                choice: None,
870                task_instruction: None,
871            });
872            let wait = repository_wait(&state, None);
873            let coordination_config = state.config.coordination.clone();
874            let coordination =
875                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876                    .await?;
877            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878            if let Some(released) = &released {
879                state.event(
880                    "release",
881                    format!(
882                        "took `{branch}` over from run {}: its worktree was released: {}",
883                        crate::run::short_of(&released.old_id),
884                        released.audit
885                    ),
886                );
887            }
888            let opened = async {
889                startup.stage(&mut state, "review-preflight")?;
890                // The owner's answer to an earlier divergence question is applied
891                // here: after the release (git will not move a checked-out branch)
892                // and before the sync that would otherwise ask again.
893                state.foreign_checkout = crate::handover::preflight_review(
894                    &repo,
895                    branch,
896                    &takeover.home,
897                    &state.config.merge.remote,
898                )
899                .await?;
900                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902                }
903                if let Some(choice) = takeover.choice.as_ref()
904                    && let Err(e) =
905                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906                            .await
907                {
908                    return Err(e.context("applying the owner's answer about the diverged branch"));
909                }
910                // An earlier attempt of this task that published the task head, and
911                // whose work is the branch under review, hands that head over: the
912                // review run updates the same pull request. Without one (a legacy
913                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
914                // own head and its own open pull request is adopted as before.
915                for id in takeover.earlier.iter().rev() {
916                    if let Ok(old) = RunState::load_under(id, &takeover.home)
917                        && let Some(head) = old.pr_head.clone()
918                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919                    {
920                        state.pr_head = Some(head);
921                        break;
922                    }
923                }
924                drop(coordination);
925                startup.stage(&mut state, "open-review")?;
926                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927            }
928            .await;
929            if opened.is_err()
930                && let Some(released) = &released
931            {
932                // Restoration is ownership checked and best effort. Keep the original
933                // startup cause even if reacquiring coordination is no longer possible.
934                match crate::repo_lock::SeatLease::operation_with(
935                    &repo,
936                    coordination_config.as_ref(),
937                    &wait,
938                )
939                .await
940                {
941                    Ok(_coordination) => {
942                        released.restore(&repo, branch).await;
943                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
944                            .map(|old| {
945                                format!(
946                                    "predecessor released_to={:?}; candidate path exists={}",
947                                    old.released_to,
948                                    old.candidates.iter().any(|c| {
949                                        c.branch == branch && c.worktree.exists()
950                                    }),
951                                )
952                            })
953                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955                    }
956                    Err(error) => {
957                        tracing::warn!(
958                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959                            state.id
960                        );
961                        state.event(
962                            "startup-restore",
963                            format!("restoration not attempted: {error:#}"),
964                        );
965                    }
966                }
967            }
968            opened
969        }
970        .await;
971        startup.finish(&mut state, opened)?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Recover exactly the operator-selected public implementation, without touching old sources.
982    pub async fn recover_pr(
983        repo: &Path,
984        task: &str,
985        selection: crate::pr_recovery::Selection,
986        mut config: Config,
987    ) -> Result<Self> {
988        selection.configure(&mut config)?;
989        selection.check_task(task, &selection.instruction)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.origin = Some(crate::run::Origin::queue(task));
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.runs.last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.ensure_base_episode();
1417        state.record_base_sync_stop(&head);
1418        state.save()?;
1419        let fetched =
1420            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421        anyhow::ensure!(
1422            fetched.ok(),
1423            "cannot authorize base recovery without a fetched base"
1424        );
1425        let approved_tip = git::rev_parse(
1426            &state.repo,
1427            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428        )
1429        .await?;
1430        if resumption {
1431            anyhow::ensure!(
1432                crate::daemon::compatible_run_instruction(&state, &queued),
1433                "resumption instruction does not match the retained run"
1434            );
1435            let approval = queued
1436                .resumption
1437                .as_ref()
1438                .context("no operator resumption")?;
1439            anyhow::ensure!(
1440                approval.instruction == queued.instruction,
1441                "resumption approval does not match the current task instruction"
1442            );
1443            state.authorize_base_sync_resumption(task, approval)?;
1444        } else {
1445            state.authorize_base_sync(task)?;
1446        }
1447        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448        state
1449            .base_sync_recovery
1450            .authorizations
1451            .last_mut()
1452            .unwrap()
1453            .scope = Some(crate::base_sync_recovery::Scope {
1454            tip: approved_tip,
1455            instruction: queued.instruction.clone(),
1456            episode,
1457        });
1458        state.event(
1459            "base-recovery",
1460            "owner authorized a bounded base synchronization interval; fresh verification required",
1461        );
1462        state.save()?;
1463        Ok(())
1464    }
1465
1466    /// Reopen an existing run.
1467    pub fn resume(id: &str) -> Result<Self> {
1468        let state = RunState::load(id)?;
1469        if state.startup.as_ref().is_some_and(|s| !s.complete)
1470            || (state.status == RunStatus::Prep
1471                && state.instruction.trim().is_empty()
1472                && state.candidates.is_empty()
1473                && state.prep_complete == Some(false))
1474        {
1475            bail!(
1476                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477            );
1478        }
1479        crate::pr_head::ensure_owner(&state)?;
1480        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481            bail!(
1482                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483            );
1484        }
1485        if let Some(to) = &state.released_to {
1486            bail!(
1487                "run {} cannot be resumed: its worktree was released to run {}",
1488                state.short(),
1489                crate::run::short_of(to)
1490            );
1491        }
1492        let roles = state.config.resolve_roles()?;
1493        if let Some(id) = &state.config.graph.selected_implementer
1494            && state.candidates.iter().any(|c| c.agent != *id)
1495        {
1496            bail!(
1497                "replacement implementer pin does not match the saved candidates; task must stay held"
1498            );
1499        }
1500        let max_parallel = state.config.graph.max_parallel.max(1);
1501        Ok(Self {
1502            state,
1503            roles,
1504            sem: Arc::new(Semaphore::new(max_parallel)),
1505            pause: Pause::new(),
1506            interrupt: Pause::new(),
1507        })
1508    }
1509
1510    /// Walk the graph to a terminal state, skipping nodes already recorded.
1511    ///
1512    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1513    /// the phone - ends here, so this is the one place a run that ended
1514    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1515    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1516    pub async fn execute(&mut self) -> Result<()> {
1517        anyhow::ensure!(
1518            self.state.startup.as_ref().is_none_or(|s| s.complete),
1519            "startup incomplete; inspect retained evidence before explicit recovery"
1520        );
1521        if let Some(selection) = &self.state.pr_recovery
1522            && (selection.source_proof.is_some() || selection.source_run.is_some())
1523        {
1524            Box::pin(selection.verify_source(&self.state.repo)).await?;
1525        }
1526        // Queue claims cover daemon work; this also serializes manual drivers
1527        // consuming a repair answer, including the new review and decision.
1528        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529            || self.state.land_approval.is_some()
1530            || self.state.merge_decision.is_some()
1531            || !self.state.merge_repairs.is_empty()
1532            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533        {
1534            let claim = FixClaim::acquire(&self.state.dir())?;
1535            let disk = RunState::load(&self.state.id)?;
1536            let checkpoint = |s: &RunState| {
1537                (
1538                    s.merge_repairs.len(),
1539                    s.merge_repairs
1540                        .last()
1541                        .map(|r| (r.question.clone(), r.phase)),
1542                    s.land_approval
1543                        .as_ref()
1544                        .map(|a| (a.question.clone(), a.head.clone())),
1545                )
1546            };
1547            if checkpoint(&disk) != checkpoint(&self.state) {
1548                bail!(
1549                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550                );
1551            }
1552            Some(claim)
1553        } else {
1554            None
1555        };
1556        let mut execution = self.claim_driver().await?;
1557        let runner = &mut *execution.runner;
1558        let wait = runner.repository_wait();
1559        // Keep the graph's large future off the callers' stack, including
1560        // standalone task attempts running on a default-sized test thread.
1561        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562        let result = match result {
1563            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564                record_repository_block(&mut runner.state, &e);
1565                runner.park_here()?;
1566                Ok(())
1567            }
1568            other => other,
1569        };
1570        execution.driver.finish(&mut runner.state).await?;
1571        let ended = if result.is_err() {
1572            Some({
1573                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574                if let Err(error) = &result {
1575                    notice.message = format!(
1576                        "Run {} stopped with an error: {}",
1577                        runner.state.id,
1578                        error.to_string().lines().next().unwrap_or("unknown error")
1579                    );
1580                }
1581                notice
1582            })
1583        } else {
1584            crate::notices::run_ended(&runner.state)
1585        };
1586        if let Some(notice) = ended {
1587            crate::notices::raise(notice);
1588        }
1589        result
1590    }
1591
1592    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594    }
1595
1596    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597        let wait = self.repository_wait();
1598        coordinate_operation(&mut self.state, cwd, &wait).await
1599    }
1600
1601    /// Validate ownership and persist the driver claim in one critical section.
1602    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603        let owner = match crate::pr_head::owner_lock_wait(
1604            &crate::run::home(),
1605            &self.state.id,
1606            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607        )
1608        .await
1609        {
1610            Ok(owner) => owner,
1611            Err(error) => {
1612                let detail = format!("worktree maintenance or ownership update: {error:#}");
1613                // Admission did not own run.json: retain this resource outcome
1614                // in memory for queue settlement without overwriting a concurrent
1615                // retirement or publication transaction.
1616                self.state.resource_block = Some(crate::run::ResourceBlock {
1617                    cause: crate::run::ResourceBlockCause::Other,
1618                    node: "start".into(),
1619                    kind: "worktree maintenance".into(),
1620                    detail: detail.clone(),
1621                    at: Timestamp::now(),
1622                });
1623                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624            }
1625        };
1626        let disk = RunState::load(&self.state.id)?;
1627        crate::pr_head::ensure_owner(&disk)?;
1628        let claimed =
1629            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630        // Only proven-live ownership refuses here. Unknown records (legacy runs
1631        // with no claim marker or process identity) fall through to the
1632        // exclusive execution lease taken just below, which is what actually
1633        // keeps a second driver out.
1634        anyhow::ensure!(
1635            disk.liveness(claimed) != crate::run::Liveness::Live,
1636            "run already has a live driver; wait for its execution to end"
1637        );
1638        if let Some(to) = &disk.released_to {
1639            bail!(
1640                "run {} cannot continue: its worktree was released to run {}",
1641                self.state.short(),
1642                crate::run::short_of(to)
1643            );
1644        }
1645        anyhow::ensure!(
1646            serde_json::to_value(&disk.base_sync_recovery)?
1647                == serde_json::to_value(&self.state.base_sync_recovery)?,
1648            "base recovery changed since this run was loaded; reload the saved run"
1649        );
1650        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651        let mut execution = DriverExecution {
1652            runner: self,
1653            driver,
1654        };
1655        // Declare after the execution guard: on any claim error this short
1656        // lock must drop before that guard reloads and releases ownership.
1657        let _owner = owner;
1658        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659            .await?;
1660        let state = &mut execution.runner.state;
1661        // Moving again, so it is no longer parked. Set before the walk rather
1662        // than in `resume`, so every way of re-entering the graph clears it
1663        // and a card cannot claim a run is waiting to be resumed while the
1664        // agents are already working.
1665        state.parked = false;
1666        state.resource_wait = None;
1667        state.resource_block = None;
1668        // Any seat this state still lists as answering belongs to whatever
1669        // process last drove this run — this one included, if it crashed
1670        // mid-wave. Cleared and flushed immediately, before anything else
1671        // runs, so a resume can never show a seat as live when nothing is
1672        // asking it anything yet; the node that actually dispatches the next
1673        // wave repopulates it.
1674        state.clear_active();
1675        // Recorded in the same spot, and flushed together with the clear
1676        // above: this is the pid a reader checks (`RunState::liveness`) when
1677        // no daemon claim exists to answer "is a process still driving this
1678        // run" — a plain `magi run` / `magi review` typed into a terminal
1679        // claims nothing there. Always overwritten, never only-if-absent, so
1680        // a resumed run's stale pid from a previous, possibly-dead process
1681        // can never survive into this one's own report. Unlike
1682        // `clear_active`, this changes on every single `execute()` call, so
1683        // the save below is now unconditional rather than only-if-cleared.
1684        //
1685        // `driver_started_at` is recorded in the same breath, from this same
1686        // pid, so `liveness` can tell a live pid that is genuinely still us
1687        // apart from one the OS has since handed to an unrelated process —
1688        // see that field's own doc for why the pid alone is not enough.
1689        let pid = std::process::id();
1690        state.driver_pid = Some(pid);
1691        state.driver_started_at = crate::proc::process_started_at(pid);
1692        state.driver_exited = false;
1693        state.driver_claimed = Some(true);
1694        state.begin_pass();
1695        execution.driver.bind(state);
1696        state.save()?;
1697        execution.driver.record(state)?;
1698        Ok(execution)
1699    }
1700
1701    /// No conflict checkpoint proves dispatch was never reached. End that
1702    /// consumed generation so an owner can explicitly authorize a fresh one.
1703    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1704        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1705        if let Err(error) = &result
1706            && !error.is::<crate::repo_lock::ResourceBlocked>()
1707            && self
1708                .state
1709                .repair_grants
1710                .last()
1711                .is_some_and(|g| g.conflict.is_none())
1712        {
1713            self.state.repair_grants.last_mut().unwrap().phase =
1714                crate::repair_budget::Phase::Complete;
1715            self.state.status = RunStatus::Blocked;
1716            self.state.save()?;
1717            bail!(
1718                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1719                self.state.passes.last().map_or(0, |p| p.index)
1720            );
1721        }
1722        result
1723    }
1724
1725    /// A grant owns only these repair attempts. Confirmation reviews cannot
1726    /// create another fixer allowance, and every operation is restart-checkpointed.
1727    async fn execute_granted_repair(&mut self) -> Result<()> {
1728        use crate::repair_budget::{Kind, Phase};
1729        let index = self.state.repair_grants.len() - 1;
1730        let winner = self
1731            .state
1732            .winner()
1733            .cloned()
1734            .context("repair winner missing")?;
1735        loop {
1736            let grant = self.state.repair_grants[index].clone();
1737            match grant.phase {
1738                Phase::Pending => {
1739                    if grant.spent >= grant.receipt.request.rounds {
1740                        self.state.repair_grants[index].phase = Phase::Complete;
1741                        self.state.status = RunStatus::Blocked;
1742                        self.state.save()?;
1743                        return Ok(());
1744                    }
1745                    // Reserve before dispatch. A crash can consume an attempt,
1746                    // but can never duplicate it or reset saved fixer history.
1747                    self.state.repair_grants[index].spent += 1;
1748                    self.state.repair_grants[index].phase = Phase::Fixing;
1749                    self.state.save()?;
1750                    match grant.receipt.request.kind {
1751                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1752                        Kind::Gate => {
1753                            let failed = self.state.gate.clone();
1754                            let _ = self.gate_fix_round(&winner, &failed).await?;
1755                        }
1756                        Kind::Review => {
1757                            let mut record = self
1758                                .state
1759                                .reviews
1760                                .last()
1761                                .cloned()
1762                                .context("review failure missing")?;
1763                            let round = record.round;
1764                            let step = self.run_fix_round(&winner, &mut record).await?;
1765                            self.state.reviews[round - 1] = record;
1766                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1767                                self.state.repair_grants[index].phase = Phase::Complete;
1768                                self.state.status = RunStatus::Blocked;
1769                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1770                                self.state.save()?;
1771                                return Ok(());
1772                            }
1773                            if matches!(step, FixStep::Resource) {
1774                                self.stop_for_resource()?;
1775                                return Ok(());
1776                            }
1777                            self.state.save()?;
1778                        }
1779                    }
1780                    if self.resource_halt()? {
1781                        return Ok(());
1782                    }
1783                }
1784                Phase::Fixing => {
1785                    if grant.receipt.request.kind == Kind::BaseConflict {
1786                        self.repair_base_conflict(false).await?;
1787                    }
1788                    // Interrupted fixers may have left source changes or an
1789                    // unpushed commit. Preserve them before fresh assessment.
1790                    let _coordination = self.coordinate(&winner.worktree).await?;
1791                    let rescued = git::rescue_commit(
1792                        &winner.worktree,
1793                        "magi: preserve interrupted granted repair",
1794                    )
1795                    .await?;
1796                    self.state.note_withheld("repair-grant", &rescued.withheld);
1797                    // Fresh checks must supersede even a same-head failure.
1798                    // Archive rather than erase failure evidence or counters.
1799                    self.state
1800                        .verification_history
1801                        .push(crate::run::VerificationEvidence {
1802                            superseded_by: None,
1803                            archived_at: Some(Timestamp::now()),
1804                            fixes_spent: Some(self.state.gate_fixes.len()),
1805                            gate_head: self.state.gate_head.take(),
1806                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1807                            gate: std::mem::take(&mut self.state.gate),
1808                            e2e: self.state.head_e2e.take(),
1809                        });
1810                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1811                        self.state
1812                            .verification_history
1813                            .push(crate::run::VerificationEvidence {
1814                                superseded_by: None,
1815                                archived_at: Some(Timestamp::now()),
1816                                fixes_spent: Some(self.state.gate_fixes.len()),
1817                                gate_head: None,
1818                                gate_ran: false,
1819                                gate: Vec::new(),
1820                                e2e: Some(e2e),
1821                            });
1822                    }
1823                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1824                    self.state.repair_grants[index].head =
1825                        git::rev_parse(&winner.worktree, "HEAD").await?;
1826                    self.state.repair_grants[index].phase = Phase::Reviewing;
1827                    if grant.receipt.request.kind == Kind::BaseConflict {
1828                        self.state.base_sync_recovery.stop = None;
1829                        if let Some(sync) = &mut self.state.base_sync {
1830                            sync.tip = grant
1831                                .receipt
1832                                .request
1833                                .tip
1834                                .clone()
1835                                .context("repair base tip missing")?;
1836                            sync.conflict = None;
1837                            sync.behind = 0;
1838                        }
1839                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1840                    }
1841                    self.state.save()?;
1842                }
1843                Phase::Reviewing => {
1844                    self.review_loop_pass().await?;
1845                    if self.resource_halt()? {
1846                        return Ok(());
1847                    }
1848                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1849                    let clean = self.state.reviews.len() > grant.review_start
1850                        && self.state.reviews.last().is_some_and(|r| {
1851                            r.head == head
1852                                && r.clean
1853                                && r.expected > 0
1854                                && r.answered == r.expected
1855                                && matches!(
1856                                    r.e2e_status(),
1857                                    E2eStatus::Passed | E2eStatus::NotConfigured
1858                                )
1859                        });
1860                    if !clean {
1861                        // A confirmation review carries no additional fix budget.
1862                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1863                            == Kind::Review
1864                            && grant.spent < grant.receipt.request.rounds
1865                            && self.state.reviews.last().is_some_and(|r| {
1866                                r.fix.is_none()
1867                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1868                            }) {
1869                            Phase::Pending
1870                        } else {
1871                            Phase::Complete
1872                        };
1873                        self.state.status = RunStatus::Blocked;
1874                        self.state.event(
1875                            "repair-grant",
1876                            "fresh review did not clear the repaired head; no publication",
1877                        );
1878                        self.state.save()?;
1879                        if self.state.repair_grants[index].phase == Phase::Pending {
1880                            continue;
1881                        }
1882                        return Ok(());
1883                    }
1884                    self.state.repair_grants[index].phase = Phase::Gating;
1885                    self.state.save()?;
1886                }
1887                Phase::Gating => {
1888                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1889                    self.state.reconcile_verification_head(&head);
1890                    let wait = self.repository_wait();
1891                    if self
1892                        .state
1893                        .reviews
1894                        .last()
1895                        .is_none_or(|r| r.head != head || !r.clean)
1896                    {
1897                        self.state.status = RunStatus::Blocked;
1898                        self.state.repair_grants[index].phase = Phase::Complete;
1899                        self.state.event(
1900                            "repair-grant",
1901                            "head changed since fresh review; repair held",
1902                        );
1903                        self.state.save()?;
1904                        return Ok(());
1905                    }
1906                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1907                        return Ok(());
1908                    }
1909                    let outcomes = self.run_gate(&winner).await?;
1910                    if verify_inconclusive(&outcomes) {
1911                        return Ok(());
1912                    }
1913                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1914                    if after != head {
1915                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1916                        self.state.repair_grants[index].head = after;
1917                        self.state.repair_grants[index].phase = Phase::Reviewing;
1918                        self.state.status = RunStatus::Blocked;
1919                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1920                        self.state.save()?;
1921                        return Ok(());
1922                    }
1923                    let passed = outcomes.iter().all(CommandOutcome::ok);
1924                    // Follow-up grants owe the same guarded checkout delivery
1925                    // as the conflict repair that introduced the new head.
1926                    if passed
1927                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1928                    {
1929                        return Ok(());
1930                    }
1931                    self.state.repair_grants[index].succeeded = passed;
1932                    self.state.gate = outcomes;
1933                    self.state.gate_ran = true;
1934                    self.state.status = if passed {
1935                        RunStatus::Gating
1936                    } else {
1937                        RunStatus::Blocked
1938                    };
1939                    self.state.repair_grants[index].phase = if !passed
1940                        && grant.receipt.request.kind == Kind::Gate
1941                        && grant.spent < grant.receipt.request.rounds
1942                    {
1943                        Phase::Pending
1944                    } else {
1945                        Phase::Complete
1946                    };
1947                    if passed && self.state.config.merge.mode == MergeMode::Pr {
1948                        // Persist publication debt with the successful checkpoint.
1949                        self.state.pr_publication_pending = true;
1950                    } else if passed {
1951                        self.state.status = RunStatus::Ready;
1952                    }
1953                    self.state.save()?;
1954                    if !passed {
1955                        continue;
1956                    }
1957                    // Existing PR ownership and push lease stay in place. The
1958                    // normal publisher owes the new reviewed/gated SHA, and
1959                    // land's fresh owner approval remains mandatory.
1960                    if self.state.config.merge.mode != MergeMode::Pr {
1961                        self.state.status = RunStatus::Ready;
1962                        self.state.event(
1963                            "repair-grant",
1964                            "repair verified locally; grant carries no merge authorization",
1965                        );
1966                        self.state.save()?;
1967                        return Ok(());
1968                    }
1969                    return self.merge().await;
1970                }
1971                Phase::Complete => return Ok(()),
1972            }
1973        }
1974    }
1975
1976    async fn execute_graph(&mut self) -> Result<()> {
1977        // Even a terminal run may reenter through an explicit recovery action.
1978        // Restore before prep shortcuts, saved conversations or graph nodes use
1979        // a disposable path; GC may already have removed its original objects.
1980        let archive_home = crate::run::home().canonicalize()?;
1981        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1982            let repo = self.state.repo.clone();
1983            let _coordination = self.coordinate(&repo).await?;
1984            ensure_setup_config(&mut self.state, &repo).await?;
1985            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1986        }
1987        self.state.configure_recovery_resume()?;
1988        if self
1989            .state
1990            .repair_grants
1991            .last()
1992            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1993        {
1994            return self.execute_granted_repair().await;
1995        }
1996        if self
1997            .state
1998            .repair_grants
1999            .last()
2000            .is_some_and(|g| !g.succeeded)
2001        {
2002            self.state.status = RunStatus::Blocked;
2003            self.state.save()?;
2004            return Ok(());
2005        }
2006        // Preservation may invalidate an old empty classification. Resolve it
2007        // before any resume shortcut can reuse a verdict or publish a branch.
2008        self.persist_candidates().await?;
2009        // A saved base stop outranks stage markers, including legacy Voting.
2010        // Recovery never reenters implementation, deliberation or voting.
2011        if self.state.base_sync_blocked() {
2012            self.sync_to_base().await?;
2013            if !self.state.base_sync_blocked()
2014                && self.state.status != RunStatus::AlreadyInBase
2015                && !self.resource_halt()?
2016            {
2017                self.finish_after_tally().await?;
2018            }
2019            return Ok(());
2020        }
2021        if self
2022            .state
2023            .merge_decision
2024            .as_ref()
2025            .is_some_and(|decision| !decision.closed)
2026            && self.state.status != RunStatus::Landing
2027        {
2028            if crate::merge_decision::resumable_hold(&self.state) {
2029                self.state.merge = Some(MergeOutcome {
2030                    mode: MergeMode::Pr,
2031                    ok: false,
2032                    detail: self
2033                        .state
2034                        .merge_decision
2035                        .as_ref()
2036                        .unwrap()
2037                        .target
2038                        .pr
2039                        .clone(),
2040                    empty: false,
2041                });
2042                self.run_land().await?;
2043                self.settle_questions();
2044                return Ok(());
2045            }
2046            bail!(
2047                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2048            );
2049        }
2050        if self.state.config.graph.selected_implementer.is_some()
2051            && self.state.status == RunStatus::Blocked
2052            && self.state.tally.is_none()
2053            && self.state.candidate_preservation.is_empty()
2054            && self.state.interrupted_implementations.is_empty()
2055        {
2056            bail!(
2057                "selected implementation is held; explicitly release the owning task to retry it"
2058            );
2059        }
2060
2061        if self.state.merge_repairs.last().is_some_and(|r| {
2062            matches!(
2063                r.phase,
2064                crate::run::MergeRepairPhase::Requested
2065                    | crate::run::MergeRepairPhase::Fixing
2066                    | crate::run::MergeRepairPhase::Reviewing
2067            )
2068        }) {
2069            if self.repair_merge_review().await? {
2070                self.run_land().await?;
2071            }
2072            self.settle_questions();
2073            return Ok(());
2074        }
2075        // Publication already passed review and gate. Resume only that node:
2076        // syncing to a newer base here could change the verified commit.
2077        if self.state.pr_publication_pending {
2078            self.merge().await?;
2079            if self.resource_halt()? {
2080                return Ok(());
2081            }
2082            self.state.save()?;
2083            self.settle_questions();
2084            return Ok(());
2085        }
2086        // A run that already lost its quorum never resumes into the verdict
2087        // machinery: `deliberate` and `vote` would otherwise clobber the
2088        // stalled marker back to Voting and the run would keep going past a
2089        // verdict that is no longer trustworthy. Everything already recorded is
2090        // kept, so the run stays resumable (or foldable) for a human to pick up.
2091        //
2092        // On --resume the run gets one chance to repair itself: the seats a
2093        // rate limit took out are re-asked. If their quota has since reset and
2094        // the quorum is restored, the run picks up and finishes; otherwise it
2095        // stays stale and still-resumable for a later retry. If it does not
2096        // recover, the returned status stays `Stalled` and nothing was
2097        // clobbered (the recovery only mutates entries for the lost seats).
2098        if self.state.status == RunStatus::Stalled {
2099            if self.recover_stall().await? {
2100                self.finish_after_tally().await?;
2101            } else {
2102                // Still below quorum: persist the marker and stay resumable.
2103                self.state.save()?;
2104            }
2105            return Ok(());
2106        }
2107        // A run parked inside `land` - watching CI, mid fix-round, or
2108        // waiting on the owner's merge approval - resumes directly into it,
2109        // never back through `prep`. Everything before `merge` already
2110        // concluded; that is the only way `status` reaches `Landing` in the
2111        // first place. Re-walking `review_loop` first would also be actively
2112        // wrong: its own status recomputation (see its doc) treats any
2113        // clean round as reason to set `status` to `Gating`, which would
2114        // clobber this marker before `merge` ever ran, and this run would
2115        // never find its way back into `land` at all.
2116        if self.state.status == RunStatus::Landing
2117            || crate::handover::landing_checkpoint(&self.state)
2118        {
2119            self.state.land_sync_pending = true;
2120            self.state.save()?;
2121            self.run_land().await?;
2122            // `run_land` may have settled the run right here - CI came back
2123            // green and the PR merged, say - without ever passing back
2124            // through `merge`'s own trailing call. Whatever it left `status`
2125            // as is what this has to read.
2126            self.settle_questions();
2127            return Ok(());
2128        }
2129        // A run parked at `merge` on the owner's word about a withheld PR text
2130        // resumes straight into `merge`. Re-walking the tail would sync to the
2131        // base again (possibly rebasing, possibly spending the fixer) after the
2132        // gate had passed, and then push a tree nobody reviewed or gated.
2133        if self.state.github_text.is_some()
2134            && self.state.merge.is_none()
2135            && self.state.status == RunStatus::Gating
2136        {
2137            self.merge().await?;
2138            if self.resource_halt()? {
2139                return Ok(());
2140            }
2141            self.state.save()?;
2142            return Ok(());
2143        }
2144        if let Err(e) = self.prep().await {
2145            if e.is::<crate::repo_lock::ResourceBlocked>() {
2146                return Err(e);
2147            }
2148            self.state.status = RunStatus::Blocked;
2149            self.state
2150                .prep_failure
2151                .get_or_insert_with(|| format!("{e:#}"));
2152            self.state
2153                .event("prep", format!("preparation stopped: {e:#}"));
2154            self.state.save()?;
2155            return Err(e);
2156        }
2157        if self.park_here()? {
2158            return Ok(());
2159        }
2160        self.advise().await?;
2161        if self.park_here()? {
2162            return Ok(());
2163        }
2164        self.implement().await?;
2165        if self.park_here()? {
2166            return Ok(());
2167        }
2168        // `after_implement` already saved the state and settled any open
2169        // questions when it set this; nothing later in the graph has
2170        // anything to judge.
2171        if self.state.status == RunStatus::VerifiedNoop {
2172            return Ok(());
2173        }
2174        self.judge().await?;
2175        if self.park_here()? {
2176            return Ok(());
2177        }
2178        self.deliberate().await?;
2179        if self.park_here()? {
2180            return Ok(());
2181        }
2182        self.vote().await?;
2183        if self.park_here()? {
2184            return Ok(());
2185        }
2186        self.tally()?;
2187        // A verdict that lost its quorum is not trustworthy: do not review,
2188        // gate, or merge on it. Everything already done is kept, so the run
2189        // stays resumable (or foldable); the human can replace the agent that
2190        // ran out of quota and pick it up.
2191        if self.state.status == RunStatus::Stalled {
2192            // Persist the stalled marker now — the normal end-of-execute save
2193            // below is below this early return, and without it a resumed run
2194            // would reload a pre-tally status and keep going.
2195            self.state.save()?;
2196            return Ok(());
2197        }
2198        self.finish_after_tally().await?;
2199        Ok(())
2200    }
2201
2202    /// A wave could not start its agents because a shared resource stayed busy
2203    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2204    /// resumable. Unstarted seats have no result; completed peers retain
2205    /// their saved candidate state.
2206    fn stop_for_resource(&mut self) -> Result<()> {
2207        // Resource acquisition observes cancellation inside a node. Complete
2208        // the same durable park as every other node boundary before returning.
2209        if self.pause.parked() || self.interrupt.parked() {
2210            self.park_here()?;
2211            return Ok(());
2212        }
2213        if self.state.resource_block.is_some() {
2214            self.state.status = RunStatus::Blocked;
2215        }
2216        self.state.save()
2217    }
2218
2219    /// Stop walking the graph if a node ended on a resource block.
2220    fn resource_halt(&mut self) -> Result<bool> {
2221        if self.state.resource_block.is_none() {
2222            return Ok(false);
2223        }
2224        self.stop_for_resource()?;
2225        Ok(true)
2226    }
2227
2228    /// Park here if asked to, recording it in the run's own timeline.
2229    ///
2230    /// Returns whether the caller should stop walking the graph. The state is
2231    /// saved either way by the node that just finished; this adds the event so
2232    /// the operator's card says why a run that is neither finished nor moving
2233    /// is sitting where it is.
2234    fn park_here(&mut self) -> Result<bool> {
2235        let parking = self.pause.parked() || self.interrupt.parked();
2236        if !parking && self.state.resource_block.is_some() {
2237            self.stop_for_resource()?;
2238            return Ok(true);
2239        }
2240        // Either handle asking is enough - see `Pause`'s own doc for why
2241        // they are never the same one. `interrupt` is checked second so a
2242        // reason it carries is preferred in the message below over a plain
2243        // shutdown park racing it at the same boundary.
2244        if !self.pause.parked() && !self.interrupt.parked() {
2245            return Ok(false);
2246        }
2247        if self.state.parked {
2248            return Ok(true);
2249        }
2250        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2251        if let Some(block) = &mut self.state.resource_block
2252            && matches!(
2253                block.cause,
2254                crate::run::ResourceBlockCause::Cancelled { .. }
2255            )
2256        {
2257            block.cause = crate::run::ResourceBlockCause::Cancelled {
2258                park: cause.clone(),
2259            };
2260        }
2261        let why = match cause.as_ref().map(ParkCause::label) {
2262            Some(reason) => format!(
2263                "parked after `{}` ({reason}) — resume to carry on from here",
2264                self.state.status.as_str()
2265            ),
2266            None => format!(
2267                "parked after `{}` — resume to carry on from here",
2268                self.state.status.as_str()
2269            ),
2270        };
2271        self.state.event("park", why);
2272        self.state
2273            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2274        self.state.parked = true;
2275        self.state.save()?;
2276        Ok(true)
2277    }
2278
2279    /// Only a new explicit operator approval may retry failed preparation.
2280    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2281        if self.state.prep_failure.is_none() {
2282            return Ok(());
2283        }
2284        let failed_at = self
2285            .state
2286            .events
2287            .iter()
2288            .rev()
2289            .find(|event| event.node == "prep")
2290            .map_or(self.state.updated_at, |event| event.at);
2291        anyhow::ensure!(
2292            approval_at > failed_at,
2293            "preparation retry requires a new explicit release after the failure"
2294        );
2295        self.state.prep_failure = None;
2296        self.state.prep_complete = Some(false);
2297        self.state
2298            .event("prep-retry", "operator approved retry of preparation");
2299        Ok(())
2300    }
2301
2302    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2303    pub fn on_pause(&mut self, pause: Pause) {
2304        self.pause = pause;
2305    }
2306
2307    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2308    /// scheduler asking this one run - and no other - to park so a task
2309    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2310    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2311    /// handle.
2312    pub fn watch_interrupt(&mut self, pause: Pause) {
2313        self.interrupt = pause;
2314    }
2315
2316    /// Abandon this run's own open questions, once `status` has actually
2317    /// settled rather than merely paused.
2318    ///
2319    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2320    /// either back up with the candidates, the review round and the seat
2321    /// sessions already on disk, so a question an implementer asked mid-round
2322    /// may still get a real answer read by a real resume. Only the statuses
2323    /// `resumable` excludes are actually final: the run merged, it reached
2324    /// `Ready` with nothing left to do, it failed outright with no
2325    /// established point to continue from, or every candidate agreed, with
2326    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2327    /// every one of those the seat that asked is gone for good, exactly like
2328    /// the run being deleted under `magi run rm` - so the same cleanup
2329    /// applies, worded for what actually happened instead of "the run was
2330    /// deleted".
2331    ///
2332    /// Best-effort and silent on success: called from every place `status`
2333    /// can land on one of those three, including ones a resumed run revisits,
2334    /// so it must cost nothing when there was nothing open to begin with.
2335    fn settle_questions(&mut self) {
2336        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2337            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2338        }
2339    }
2340
2341    /// The tail of the graph after a trustworthy tally: fold losers, review,
2342    /// gate, merge, and persist.
2343    async fn finish_after_tally(&mut self) -> Result<()> {
2344        // Before the losers are folded: their branches are what the graft
2345        // reads from.
2346        self.graft().await?;
2347        if self.resource_halt()? {
2348            return Ok(());
2349        }
2350        self.fold_losers().await?;
2351        // Before review starts, and again right before the gate: a run's
2352        // review rounds can themselves take long enough for the base to move
2353        // a second time, and the gate is the one node whose "green" gets
2354        // acted on.
2355        self.sync_to_base().await?;
2356        if self.hold_base_sync().await? || self.resource_halt()? {
2357            return Ok(());
2358        }
2359        if self.state.status == RunStatus::AlreadyInBase {
2360            return Ok(());
2361        }
2362        self.review_loop().await?;
2363        if self.resource_halt()? {
2364            return Ok(());
2365        }
2366        self.sync_to_base().await?;
2367        if self.hold_base_sync().await? || self.resource_halt()? {
2368            return Ok(());
2369        }
2370        if self.state.status == RunStatus::AlreadyInBase {
2371            return Ok(());
2372        }
2373        self.gate().await?;
2374        self.merge().await?;
2375        if self.resource_halt()? {
2376            return Ok(());
2377        }
2378        self.state.save()?;
2379        Ok(())
2380    }
2381
2382    // ---------------------------------------------------------------- prep
2383
2384    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2385    /// run with the step and its output in the event; it is never swallowed.
2386    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2387    /// resume starts over instead of finding candidates and missing seats.
2388    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2389        let result = match ensure_setup_config(&mut self.state, repo).await {
2390            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2391            Err(e) => Err(e),
2392        };
2393        if let Err(e) = result {
2394            git::worktree_remove_clean(repo, wt).await.ok();
2395            if in_prep {
2396                self.rollback_prep(repo).await;
2397            }
2398            self.state.status = RunStatus::Blocked;
2399            self.state.event(
2400                "setup",
2401                format!("worktree setup failed in {}: {e:#}", wt.display()),
2402            );
2403            self.state.save()?;
2404            return Err(e);
2405        }
2406        Ok(())
2407    }
2408
2409    /// Undo everything `prep` created, so a resume re-runs it whole.
2410    async fn rollback_prep(&mut self, repo: &Path) {
2411        let root = self.state.worktree_root();
2412        for c in std::mem::take(&mut self.state.candidates) {
2413            if matches!(
2414                git::worktree_remove_clean(repo, &c.worktree).await,
2415                Ok(true)
2416            ) {
2417                git::branch_delete(repo, &c.branch).await.ok();
2418            } else {
2419                self.state.candidates.push(c);
2420            }
2421        }
2422        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2423        for j in 1..=seats {
2424            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2425                .await
2426                .ok();
2427        }
2428        for k in 1..=self.state.config.graph.advisors {
2429            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2430                .await
2431                .ok();
2432        }
2433    }
2434
2435    async fn prep(&mut self) -> Result<()> {
2436        if let Some(reason) = &self.state.prep_failure {
2437            bail!(
2438                "candidate preparation previously failed: {reason}; correct the source before retrying"
2439            );
2440        }
2441        if self
2442            .state
2443            .prep_complete
2444            .unwrap_or(!self.state.candidates.is_empty())
2445        {
2446            return Ok(());
2447        }
2448        let repo = self.state.repo.clone();
2449        // Includes rollback, config, refs, checkout creation and setup. No
2450        // repository mutation may precede the real shared lock acquisition.
2451        let _coordination = self.coordinate(&repo).await?;
2452        let base = self.state.base_commit.clone();
2453        let plan = self.state.sources.plan(&repo, &base).await?;
2454        self.rollback_prep(&repo).await;
2455        if !self.state.candidates.is_empty() {
2456            bail!(
2457                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2458            );
2459        }
2460        self.state.prep_complete = Some(false);
2461        self.state.status = RunStatus::Prep;
2462        self.state.save()?;
2463        let repo = self.state.repo.clone();
2464        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2465        let root = self.state.worktree_root();
2466        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2467
2468        // The hook is the write-time half of the blindness contract; the
2469        // presentation filter in `blind` is the half that cannot be bypassed.
2470        let hooks_dir = self.state.dir().join("hooks");
2471        if self.state.config.blind.commit_msg_hook {
2472            std::fs::create_dir_all(&hooks_dir)
2473                .with_context(|| format!("create {}", hooks_dir.display()))?;
2474            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2475            let path = hooks_dir.join("commit-msg");
2476            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2477            make_executable(&path)?;
2478            // Ref-counted rather than a plain idempotent set: with more than
2479            // one run able to be in flight in the same repository at once
2480            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2481            // true?" check cannot tell "another run of mine still needs
2482            // this" from "nobody does", and the run that happens to finish
2483            // first would disable the hook out from under a sibling still
2484            // relying on it.
2485            git::acquire_worktree_config(&repo).await?;
2486            self.state.enabled_worktree_config = true;
2487        }
2488
2489        // `[worktree] setup` hides its products per worktree, which needs the
2490        // same per-worktree config the hook does. Held until fold, like it.
2491        ensure_setup_config(&mut self.state, &repo).await?;
2492
2493        for (index, (spec, label)) in self
2494            .roles
2495            .implementers
2496            .clone()
2497            .into_iter()
2498            .zip(labels)
2499            .enumerate()
2500        {
2501            let branch = self.state.branch_for(label);
2502            let worktree = root.join(format!("cand-{label}"));
2503            // Persist intent before Git can leave a partially created tree.
2504            // Candidate registration follows setup, so it cannot inventory
2505            // crashes or refusals during creation itself.
2506            self.state.event(
2507                "prep-intent",
2508                serde_json::json!({
2509                    "branch": branch,
2510                    "worktree": worktree,
2511                    "start": start,
2512                    "picks": plan.picks,
2513                })
2514                .to_string(),
2515            );
2516            self.state.save()?;
2517            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2518            if self.state.config.blind.commit_msg_hook {
2519                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2520            }
2521            git::local_exclude(&worktree, "/.magi/").await?;
2522            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2523                // Nothing of this prep may survive: a resume must start over,
2524                // not find half the candidates and skip the rest.
2525                git::worktree_remove_clean(&repo, &worktree).await.ok();
2526                git::branch_delete(&repo, &branch).await.ok();
2527                self.rollback_prep(&repo).await;
2528                self.state.status = RunStatus::Blocked;
2529                self.state
2530                    .event("prep", format!("worktree setup failed: {e:#}"));
2531                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2532                self.state.save()?;
2533                return Err(e);
2534            }
2535            self.state.candidates.push(Candidate {
2536                index,
2537                label,
2538                agent: spec.id.clone(),
2539                branch,
2540                worktree,
2541                summary: String::new(),
2542                stat: String::new(),
2543                files: 0,
2544                commits: 0,
2545                empty: false,
2546                failed: None,
2547                verified_noop: None,
2548                duration_ms: 0,
2549                folded: false,
2550            });
2551            self.state.save()?;
2552            for pick in &plan.picks {
2553                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2554                {
2555                    self.state.status = RunStatus::Blocked;
2556                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2557                    self.state
2558                        .event("prep", format!("cannot apply referenced commit: {e}"));
2559                    self.state.save()?;
2560                    return Err(e);
2561                }
2562            }
2563        }
2564
2565        for j in 1..=self.roles.judges.len() {
2566            let wt = root.join(format!("judge-{j}"));
2567            if !wt.exists() {
2568                git::worktree_add_detached(&repo, &wt, &base).await?;
2569                self.setup_seat_worktree(&repo, &wt, true).await?;
2570            }
2571        }
2572
2573        // Disposable, detached checkouts for the design-deliberation stage's
2574        // advisor seats — the same shape as the judges' above, at the same
2575        // base commit, since advisors also only ever read. Sized off the
2576        // configured count directly rather than a resolved roster: unlike
2577        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2578        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2579        // `prep` has no `ResolvedRoles` field to read a count from here.
2580        if self.state.config.graph.advise {
2581            for k in 1..=self.state.config.graph.advisors {
2582                let wt = root.join(format!("advisor-{k}"));
2583                if !wt.exists() {
2584                    git::worktree_add_detached(&repo, &wt, &base).await?;
2585                    self.setup_seat_worktree(&repo, &wt, true).await?;
2586                }
2587            }
2588        }
2589
2590        // A judge cannot tell it is looking at its own patch — the seats keep
2591        // separate conversations — but a panel that shares agents with the
2592        // field is less independent than it looks, and that is worth saying out
2593        // loud once per run rather than leaving it in the config.
2594        let authors: Vec<&str> = self
2595            .roles
2596            .implementers
2597            .iter()
2598            .map(|a| a.id.as_str())
2599            .collect();
2600        let overlap: Vec<String> = self
2601            .roles
2602            .judges
2603            .iter()
2604            .enumerate()
2605            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2606            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2607            .collect();
2608        if !overlap.is_empty() {
2609            let note = format!(
2610                "{} also authored a candidate; blind, but the panel is less \
2611                 independent than {} distinct agents would be",
2612                overlap.join(", "),
2613                self.roles.judges.len()
2614            );
2615            self.state.event("prep", note);
2616        }
2617
2618        self.state.event(
2619            "prep",
2620            format!(
2621                "{} candidates, {} judges, base {} ({})",
2622                self.state.candidates.len(),
2623                self.roles.judges.len(),
2624                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2625                self.state.base_branch
2626            ),
2627        );
2628        self.state.prep_complete = Some(true);
2629        self.state.status = RunStatus::Implementing;
2630        self.state.save()?;
2631        Ok(())
2632    }
2633
2634    // -------------------------------------------------------------- advise
2635
2636    /// The design-deliberation stage: independent, read-only advisor seats
2637    /// each sketch a design before any implementer touches the repository,
2638    /// and (when at least one produced a usable proposal) a synthesis seat
2639    /// blends them into a brief `implement` carries in every candidate's
2640    /// prompt.
2641    ///
2642    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2643    /// advisors` is the proposal count. Everything here is best-effort and
2644    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2645    /// that cannot reach quota, or a synthesis seat that produced nothing
2646    /// usable all leave `implement` exactly as it was before this stage
2647    /// existed — the task instruction alone — rather than failing the whole
2648    /// competition over an enrichment stage. Every outcome is still recorded
2649    /// as an event, so a run that got nothing from this stage says why.
2650    ///
2651    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2652    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2653    /// resumed run whose stage failed would re-run it, and re-spend the
2654    /// agent calls, on every reentry before `implement`.
2655    ///
2656    /// Also skipped once any candidate shows implementation progress — the
2657    /// exact predicate `implement` itself uses to decide a candidate is no
2658    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2659    /// is not enough: a run created by an older binary that predates this
2660    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2661    /// an already-`Implementing`-or-later run under this build would
2662    /// otherwise walk straight back through `prep` (a no-op once candidates
2663    /// exist) into this node and spawn every advisor seat against worktrees
2664    /// `prep` never recreated — after implementation has already started,
2665    /// which is exactly the invariant this stage exists to guarantee.
2666    async fn advise(&mut self) -> Result<()> {
2667        let implement_untouched = self
2668            .state
2669            .candidates
2670            .iter()
2671            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2672        if !self.state.config.graph.advise || self.state.advise_attempted {
2673            return Ok(());
2674        }
2675        if !implement_untouched {
2676            self.state.event(
2677                "advise",
2678                "skipping the design-deliberation stage: at least one \
2679                 candidate already shows implementation progress, so this \
2680                 run is past the point the stage exists to run before"
2681                    .to_owned(),
2682            );
2683            self.state.advise_attempted = true;
2684            self.state.save()?;
2685            return Ok(());
2686        }
2687        let run_id = self.state.id.clone();
2688        let prompts = self.state.config.prompts.clone();
2689        let instruction = self.state.instruction.clone();
2690        let language = self.state.config.graph.language.clone();
2691        let root = self.state.worktree_root();
2692        let n = self.state.config.graph.advisors;
2693        let where_recorded = self.state.dir().join("run.json");
2694
2695        let seats = match self.state.config.advisors() {
2696            Ok(seats) if !seats.is_empty() => seats,
2697            Ok(_) => {
2698                self.state.event(
2699                    "advise",
2700                    format!(
2701                        "[graph] advisors is 0; skipping the design-deliberation \
2702                         stage and continuing without a synthesis brief (see {})",
2703                        where_recorded.display()
2704                    ),
2705                );
2706                self.state.advise_attempted = true;
2707                self.state.save()?;
2708                return Ok(());
2709            }
2710            Err(e) => {
2711                self.state.event(
2712                    "advise",
2713                    format!(
2714                        "could not resolve advisor seats ({e:#}); continuing \
2715                         without a design-deliberation brief (see {})",
2716                        where_recorded.display()
2717                    ),
2718                );
2719                self.state.advise_attempted = true;
2720                self.state.save()?;
2721                return Ok(());
2722            }
2723        };
2724
2725        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2726        let artifacts = agent::artifacts_dir(&self.state.dir());
2727        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2728
2729        let mut jobs = Vec::new();
2730        for (i, spec) in seats.iter().cloned().enumerate() {
2731            let seat_key = format!("advisor-{}", i + 1);
2732            let seat = self.seat(&seat_key, &spec.id);
2733            jobs.push(SeatJob {
2734                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2735                spec,
2736                seat,
2737                cwd: worktrees[i % worktrees.len()].clone(),
2738                timeout,
2739                allow_write: false,
2740                sessions: false,
2741                artifacts: artifacts.clone(),
2742                stem: seat_key,
2743                handover: None,
2744            });
2745        }
2746
2747        self.state.event(
2748            "advise",
2749            format!(
2750                "{} advisor seat(s) sketching a design in parallel",
2751                jobs.len()
2752            ),
2753        );
2754        let mut quota_losses = Vec::new();
2755        let cache = self.state.config.cache_dir();
2756        let ctx = WaveCtx {
2757            carry_seats: false,
2758            pauses: Some((&self.pause, &self.interrupt)),
2759            run: &run_id,
2760            node: "advise",
2761            prompts: &prompts,
2762            cache: cache.as_deref(),
2763            round: None,
2764        };
2765        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2766        let results = ask_json_wave::<Proposal>(
2767            jobs,
2768            Arc::clone(&self.sem),
2769            self.state.config.graph.retries,
2770            &advisor_roster,
2771            &ctx,
2772            &mut quota_losses,
2773            &mut self.state,
2774            &|p: &Proposal| p.validate(),
2775        )
2776        .await;
2777        if self.state.resource_block.is_some() {
2778            return self.stop_for_resource();
2779        }
2780        self.state.quota.extend(quota_losses);
2781
2782        let mut records = Vec::with_capacity(results.len());
2783        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2784            let agent_id = seat.agent.clone();
2785            self.state.seats.insert(seat.key.clone(), seat);
2786            match res {
2787                Ok((proposal, out)) => {
2788                    self.state
2789                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2790                    records.push(advise::AdvisorRecord::proposed(
2791                        i + 1,
2792                        agent_id,
2793                        proposal,
2794                        out.duration_ms,
2795                    ));
2796                }
2797                Err(e) => {
2798                    self.state.event(
2799                        "advise",
2800                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2801                    );
2802                    records.push(advise::AdvisorRecord::failed(
2803                        i + 1,
2804                        agent_id,
2805                        e.to_string(),
2806                    ));
2807                }
2808            }
2809        }
2810
2811        let mut advice = advise::Advice {
2812            records,
2813            synthesis: None,
2814        };
2815        if advice.proposals().is_empty() {
2816            self.state.event(
2817                "advise",
2818                "no advisor produced a usable proposal; continuing without a \
2819                 synthesis brief"
2820                    .to_owned(),
2821            );
2822        } else {
2823            match self
2824                .synthesize_brief(
2825                    &advice,
2826                    &instruction,
2827                    &language,
2828                    &worktrees[0],
2829                    &artifacts,
2830                    &run_id,
2831                    &prompts,
2832                    cache.as_deref(),
2833                )
2834                .await
2835            {
2836                Ok(Some(text)) => {
2837                    self.state.event(
2838                        "advise",
2839                        "synthesized a design brief for the implementer".to_owned(),
2840                    );
2841                    advice.synthesis = Some(text);
2842                }
2843                Ok(None) => {
2844                    self.state.event(
2845                        "advise",
2846                        "the synthesis seat produced nothing usable; continuing \
2847                         without a design brief"
2848                            .to_owned(),
2849                    );
2850                }
2851                Err(e) => {
2852                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2853                        return Err(e);
2854                    }
2855                    self.state.event(
2856                        "advise",
2857                        format!("could not synthesize a design brief: {e:#}"),
2858                    );
2859                }
2860            }
2861        }
2862        advise::apply_reflection(&mut advice);
2863
2864        self.state.advice = Some(advice);
2865        self.state.advise_attempted = true;
2866        self.state.save()?;
2867        Ok(())
2868    }
2869
2870    /// The synthesis seat: reads every advisor's proposal and blends them
2871    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2872    /// out of [`Runner::advise`] only for readability — it is not called
2873    /// anywhere else.
2874    ///
2875    /// Picked the same way [`crate::talk`]'s standing conversation and
2876    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2877    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2878    /// order (a claude seat, else the first runnable agent in roster order)
2879    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2880    /// in [`crate::config`] for why a dedicated field exists here at all.
2881    #[allow(clippy::too_many_arguments)]
2882    async fn synthesize_brief(
2883        &mut self,
2884        advice: &advise::Advice,
2885        instruction: &str,
2886        language: &str,
2887        cwd: &Path,
2888        artifacts: &Path,
2889        run_id: &str,
2890        prompts: &Prompts,
2891        cache: Option<&Path>,
2892    ) -> Result<Option<String>> {
2893        let chain = agent::pick_chain(
2894            &self.state.config.agents,
2895            self.state.config.roles.synthesizer.as_ref(),
2896            &agent::installed,
2897            "synthesizer",
2898        )?;
2899        let proposals = advice.proposals();
2900        let mut prompt = prompt::with_overlay(
2901            prompt::synthesize_brief(instruction, &proposals, language),
2902            prompts.overlay("advise"),
2903        );
2904        if cache.is_some() {
2905            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2906            // below — see `prompt::build_cache_note`'s doc for why telling a
2907            // read-only seat to build through the shared cache is exactly how
2908            // a sandbox's write refusal gets misread as a defect.
2909            prompt.push('\n');
2910            prompt.push_str(&prompt::build_cache_note("advise", false));
2911        }
2912        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2913        // Each id is tried once, in order; a quota hit, error or unusable
2914        // answer moves to the next. The seat is single-turn (`sessions:
2915        // false`) and the prompt is the whole context, so a fallback agent
2916        // needs nothing carried over.
2917        let mut last = None;
2918        for (n, spec) in chain.iter().enumerate() {
2919            if n > 0 {
2920                self.state
2921                    .event("advise", format!("synthesis falling back to {}", spec.id));
2922            }
2923            let mut seat = self.seat("advise-synthesis", &spec.id);
2924            let wait = self.repository_wait();
2925            let outcome = wait
2926                .scope(agent::invoke(
2927                    spec,
2928                    &mut seat,
2929                    &Invocation {
2930                        coordination: self.state.config.coordination.as_ref(),
2931                        fetched_base: None,
2932                        cwd,
2933                        prompt: &prompt,
2934                        timeout,
2935                        allow_write: false,
2936                        sessions: false,
2937                        artifacts,
2938                        stem: &if n == 0 {
2939                            "advise-synthesis".to_owned()
2940                        } else {
2941                            format!("advise-synthesis-{}", spec.id)
2942                        },
2943                        run: run_id,
2944                        node: "advise",
2945                        cache_dir: None,
2946                        attachments: &[],
2947                        writable: &[],
2948                    },
2949                ))
2950                .await;
2951            if outcome.is_ok() {
2952                self.state.seats.insert(seat.key.clone(), seat);
2953            }
2954            let advance = agent::chain_advances(&outcome);
2955            last = Some(outcome);
2956            if !advance {
2957                break;
2958            }
2959        }
2960        // Exhausted: the last attempt's result is what a single failed seat
2961        // would have produced.
2962        let out = last.expect("a chain holds at least one agent")?;
2963        if !out.usable() {
2964            return Ok(None);
2965        }
2966        let text =
2967            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2968        Ok((!text.trim().is_empty()).then_some(text))
2969    }
2970
2971    // ----------------------------------------------------------- implement
2972
2973    async fn implement(&mut self) -> Result<()> {
2974        self.persist_candidates().await?;
2975        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2976        // agent files with `magi task add` name the run that paid for it. The
2977        // prompt overlay is cloned alongside it because the waves borrow it
2978        // while `self` is mutably borrowed by the node's own bookkeeping.
2979        let run_id = self.state.id.clone();
2980        let prompts = self.state.config.prompts.clone();
2981        let todo: Vec<usize> = self
2982            .state
2983            .candidates
2984            .iter()
2985            .enumerate()
2986            .filter(|(_, c)| {
2987                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2988                    && c.failed.is_none()
2989                    && !c.empty
2990            })
2991            .map(|(i, _)| i)
2992            .collect();
2993        if todo.is_empty() {
2994            return self.after_implement();
2995        }
2996        self.state.status = RunStatus::Implementing;
2997
2998        let language = self.state.config.graph.language.clone();
2999        let github_language = self.state.config.graph.github_language.clone();
3000        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3001        let sessions = self.state.config.graph.sessions;
3002        let artifacts = agent::artifacts_dir(&self.state.dir());
3003        // The design-deliberation stage's blended brief, when `advise` found
3004        // one — carried into every implementer's prompt the same way
3005        // regardless of which candidate it is.
3006        let brief = self
3007            .state
3008            .advice
3009            .as_ref()
3010            .and_then(|a| a.synthesis.as_deref())
3011            .map(str::to_owned);
3012        let attachments = self.state.attachments.clone();
3013
3014        let mut jobs = Vec::new();
3015        for &i in &todo {
3016            let (index, label, worktree) = {
3017                let c = &self.state.candidates[i];
3018                (c.index, c.label, c.worktree.clone())
3019            };
3020            let spec = self.roles.implementers[index].clone();
3021            let seat_key = format!("impl-{label}");
3022            let seat = self.seat(&seat_key, &spec.id);
3023            let instruction = seeded_instruction(&self.state);
3024            jobs.push(SeatJob {
3025                spec,
3026                seat,
3027                prompt: prompt::implement_in(
3028                    &instruction,
3029                    &worktree.to_string_lossy(),
3030                    &language,
3031                    &github_language,
3032                    brief.as_deref(),
3033                    &attachments,
3034                ),
3035                cwd: worktree,
3036                timeout,
3037                allow_write: true,
3038                sessions,
3039                artifacts: artifacts.clone(),
3040                stem: format!("impl-{label}"),
3041                handover: None,
3042            });
3043        }
3044
3045        self.state.event(
3046            "implement",
3047            format!("{} candidates in parallel", jobs.len()),
3048        );
3049        // Kept so a seat whose CLI hung up can be asked again from the same
3050        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3051        // can update a seat's own entry once a fallback agent takes it over —
3052        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3053        // whichever agent actually answered, not the one that quota'd out.
3054        let mut sent = jobs.clone();
3055        let cache = self.state.config.cache_dir();
3056        let ctx = WaveCtx {
3057            carry_seats: false,
3058            pauses: Some((&self.pause, &self.interrupt)),
3059            run: &run_id,
3060            node: "implement",
3061            prompts: &prompts,
3062            cache: cache.as_deref(),
3063            round: None,
3064        };
3065        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3066        if !wave_blocked(&results) {
3067            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3068                .await;
3069            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3070                .await?;
3071            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3072                .await;
3073        }
3074        // Do not finalize failed candidates when the entire wave needs machine
3075        // recovery: leaving them pending resumes this node with its work intact.
3076        if !results.is_empty()
3077            && results
3078                .iter()
3079                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3080        {
3081            let losses: Vec<_> = results
3082                .iter()
3083                .map(|(_, seat, out)| {
3084                    let AgentOutcome::Quota(o) = out else {
3085                        unreachable!()
3086                    };
3087                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3088                })
3089                .collect();
3090            self.state.quota.extend(losses.clone());
3091            for (_, seat, _) in results {
3092                self.state.seats.insert(seat.key.clone(), seat);
3093            }
3094            for &i in &todo {
3095                let label = self.state.candidates[i].label;
3096                self.state.interrupted_implementations.insert(label);
3097                self.state
3098                    .candidate_preservation
3099                    .insert(label, "machine outage awaiting persistence".into());
3100            }
3101            // Preserve partial edits too, while keeping every interrupted seat
3102            // pending even when its tree already contains an inherited patch.
3103            block_machine_outage(&mut self.state, "implement", &losses);
3104            self.state.save()?;
3105            self.persist_candidates().await?;
3106            for &i in &todo {
3107                self.state
3108                    .interrupted_implementations
3109                    .insert(self.state.candidates[i].label);
3110            }
3111            return self.stop_for_resource();
3112        }
3113        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3114
3115        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3116            && results
3117                .iter()
3118                .any(|(_, _, out)| FailClass::of(out).is_some());
3119        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3120            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3121                // Executed turns interrupted by disk pressure already carry
3122                // a preservation barrier from wave. Admission-only waits do
3123                // not. Keep the interrupted seat's session for recovery too.
3124                if self
3125                    .state
3126                    .candidate_preservation
3127                    .contains_key(&self.state.candidates[i].label)
3128                {
3129                    self.state.seats.insert(seat.key.clone(), seat);
3130                }
3131                continue;
3132            }
3133            let seat_key = seat.key.clone();
3134            // A quota fallback (`resume_seat_handovers`) may have handed this
3135            // seat to a different agent than the one `prep` recorded on the
3136            // candidate; the stats tables and any later fixer-defaults-to-
3137            // winner's-author lookup must credit whoever actually answered —
3138            // unless every fallback also quota'd out, in which case nobody
3139            // actually answered and crediting the last agent tried would
3140            // erase every earlier agent's own quota loss from the stats
3141            // tables instead of just this one seat's.
3142            let agent = seat.agent.clone();
3143            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3144            self.state.seats.insert(seat.key.clone(), seat);
3145            let label = self.state.candidates[i].label;
3146            self.state.interrupted_implementations.remove(&label);
3147
3148            let (summary, duration, failed, verified_claim) = match out {
3149                AgentOutcome::Ok(o) => {
3150                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3151                    let failed = (!o.usable()).then(|| {
3152                        if o.timed_out {
3153                            "agent timed out".to_owned()
3154                        } else {
3155                            format!(
3156                                "agent exited with {}",
3157                                crate::notices::exit_status(o.exit_code)
3158                            )
3159                        }
3160                    });
3161                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3162                    (text, o.duration_ms, failed, verified_claim)
3163                }
3164                // Left un-resumed by `resume_undelivered` (a dirty tree
3165                // already rescues the work, or there was no session left to
3166                // resume into) — reported like the ordinary failure it is,
3167                // never as if `o.text` (the CLI's raw error JSON) were an
3168                // answer.
3169                AgentOutcome::Dropped(o) => {
3170                    let why = o
3171                        .dropped
3172                        .as_ref()
3173                        .map(|d| d.why.as_str())
3174                        .unwrap_or("the CLI ended the stream without delivering its answer");
3175                    (
3176                        String::new(),
3177                        o.duration_ms,
3178                        Some(format!("the CLI dropped the stream ({why})")),
3179                        None,
3180                    )
3181                }
3182                AgentOutcome::Quota(o) => {
3183                    self.state.quota.push(QuotaLoss::from_report(
3184                        seat_key,
3185                        "implement",
3186                        o.quota.as_ref(),
3187                    ));
3188                    (
3189                        String::new(),
3190                        o.duration_ms,
3191                        Some("rate limited (quota); produced no change".to_owned()),
3192                        None,
3193                    )
3194                }
3195                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3196                    (String::new(), 0, Some(e), None)
3197                }
3198            };
3199
3200            // Save every completed turn before any fallible Git operation. A failed
3201            // rescue must not erase the reply or prevent collection of other seats.
3202            let c = &mut self.state.candidates[i];
3203            if !exhausted_the_fallback_chain {
3204                c.agent = agent;
3205            }
3206            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3207            c.duration_ms = duration;
3208            c.failed = failed;
3209            c.verified_noop = verified_claim;
3210            self.state
3211                .candidate_preservation
3212                .insert(label, "awaiting persistence".to_owned());
3213        }
3214        self.state.save()?;
3215        self.persist_candidates().await?;
3216
3217        if resource_blocked {
3218            return self.stop_for_resource();
3219        }
3220        if selected_failed {
3221            self.state.status = RunStatus::Blocked;
3222            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3223            self.state.save()?;
3224            bail!(
3225                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3226            );
3227        }
3228        self.after_implement()
3229    }
3230
3231    /// Retry only persistence after a completed implementation turn.
3232    async fn persist_candidates(&mut self) -> Result<()> {
3233        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3234        for label in labels {
3235            let i = self
3236                .state
3237                .candidates
3238                .iter()
3239                .position(|c| c.label == label)
3240                .context("pending candidate is missing")?;
3241            let c = self.state.candidates[i].clone();
3242            let interrupted = self.state.interrupted_implementations.contains(&label);
3243            let result = async {
3244                let wait = crate::repo_lock::WaitPolicy::new(
3245                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3246                    || false,
3247                );
3248                let _coordination =
3249                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3250                // Best effort evidence requires no index writes. A successful snapshot
3251                // never authorizes removal of a tree whose rescue failed.
3252                if let Err(e) = git::snapshot_uncommitted(
3253                    &c.worktree,
3254                    &self
3255                        .state
3256                        .dir()
3257                        .join("artifacts")
3258                        .join(format!("candidate-{label}-recovery")),
3259                )
3260                .await
3261                {
3262                    self.state.event(
3263                        "preserve",
3264                        format!("candidate {label} snapshot incomplete: {e:#}"),
3265                    );
3266                }
3267                let rescued = git::rescue_commit(
3268                    &c.worktree,
3269                    &format!("magi: candidate {label} (uncommitted work)"),
3270                )
3271                .await?;
3272                self.state.note_withheld("implement", &rescued.withheld);
3273                let base = &self.state.base_commit;
3274                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3275                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3276                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3277                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3278                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3279                if c.empty && commits > 0 && !patch.trim().is_empty()
3280                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3281                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3282                        || self.state.tally.is_some())
3283                {
3284                    // Keep the old classification and pending marker so every
3285                    // retry stops too. Retain decisions as recovery evidence;
3286                    // only a supported recovery operation may replace them.
3287                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3288                }
3289                let c = &mut self.state.candidates[i];
3290                c.commits = commits;
3291                c.stat = stat;
3292                c.files = files;
3293                let no_changes = commits == 0 || patch.trim().is_empty();
3294                c.empty = no_changes && !interrupted;
3295                if interrupted && no_changes {
3296                    c.failed = None;
3297                    c.verified_noop = None;
3298                }
3299                if !c.empty {
3300                    c.failed = None;
3301                    c.verified_noop = None;
3302                }
3303                self.state.event(
3304                    "preserve",
3305                    format!("candidate {label}: {files} files, {commits} commits"),
3306                );
3307                Ok::<_, anyhow::Error>(())
3308            }
3309            .await;
3310            match result {
3311                Ok(()) => {
3312                    self.state.candidate_preservation.remove(&label);
3313                    if !interrupted || self.state.candidates[i].files > 0 {
3314                        self.state.interrupted_implementations.remove(&label);
3315                    }
3316                }
3317                Err(e) => {
3318                    let reason = format!("candidate {label} preservation failed: {e:#}");
3319                    self.state.event("preserve", &reason);
3320                    self.state.candidate_preservation.insert(label, reason);
3321                }
3322            }
3323            if self.state.candidate_preservation.is_empty()
3324                && self.state.status == RunStatus::Blocked
3325                && self.state.candidates.iter().all(|c| c.failed.is_none())
3326            {
3327                // Persist the recovered phase with the cleared pending marker;
3328                // a crash between saves must not turn this into a selected-seat
3329                // failure hold on the next resume.
3330                self.state.status = RunStatus::Implementing;
3331                self.state.event(
3332                    "preserve",
3333                    "candidate persistence recovered; implementation can continue",
3334                );
3335            }
3336            self.state.save()?;
3337        }
3338        if !self.state.candidate_preservation.is_empty() {
3339            self.state.status = RunStatus::Blocked;
3340            self.state.save()?;
3341            bail!(
3342                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3343            );
3344        }
3345        Ok(())
3346    }
3347
3348    /// Ask again, once, for work a CLI did and then failed to hand over.
3349    ///
3350    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3351    /// status with an empty response and a usage report showing output tokens,
3352    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3353    /// seven minutes and 14,267 output tokens that arrived as an empty
3354    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3355    ///
3356    /// Two conditions, and both matter:
3357    ///
3358    /// - **Only when the tree is untouched.** Often the agent has already
3359    ///   written its files and only the closing message was lost; the rescue
3360    ///   commit below picks that up and there is nothing to ask for. Re-asking
3361    ///   then would pay for a second implementation of work already on disk.
3362    /// - **Once.** A CLI that drops one stream can drop the next, and this
3363    ///   node is the most expensive in the graph.
3364    ///
3365    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3366    /// dropped reply still carried its `conversation_id`, so the seat is asked
3367    /// to finish what it was doing rather than sent the whole task again. It
3368    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3369    /// node's - for the same reason a re-ranked judge does: restating finished
3370    /// work is not the work.
3371    ///
3372    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3373    /// same way until it resets, while an abandoned conversation is still
3374    /// there to be picked up.
3375    async fn resume_undelivered(
3376        &mut self,
3377        results: &mut [(usize, SeatState, AgentOutcome)],
3378        sent: &[SeatJob],
3379        prompts: &Prompts,
3380        run_id: &str,
3381    ) {
3382        for (wi, seat, out) in results.iter_mut() {
3383            let Some(dropped) = (match &*out {
3384                AgentOutcome::Dropped(o) => o.dropped.clone(),
3385                _ => None,
3386            }) else {
3387                continue;
3388            };
3389            let Some(job) = sent.get(*wi) else { continue };
3390            // Already on disk? Then only the closing message was lost.
3391            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3392                self.state.event(
3393                    "implement",
3394                    format!(
3395                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3396                         work is in the tree",
3397                        seat.key, dropped.output_tokens, dropped.why
3398                    ),
3399                );
3400                continue;
3401            }
3402            // The re-ask only makes sense as a resume: `resume_after_drop`
3403            // says nothing about the task, trusting the seat to still hold it.
3404            // Without a session to resume — sessions disabled, or this CLI's
3405            // drop shape happened not to carry a session id — that prompt
3406            // would open a brand-new conversation with no context at all,
3407            // which is worse than leaving this as the ordinary failure it
3408            // already is.
3409            if !has_context(&job.spec, seat, job.sessions) {
3410                self.state.event(
3411                    "implement",
3412                    format!(
3413                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3414                         is no session left to resume",
3415                        seat.key, dropped.output_tokens, dropped.why
3416                    ),
3417                );
3418                continue;
3419            }
3420            self.state.event(
3421                "implement",
3422                format!(
3423                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3424                     conversation",
3425                    seat.key, dropped.output_tokens, dropped.why
3426                ),
3427            );
3428            let mut retry = job.clone();
3429            retry.seat = seat.clone();
3430            retry.prompt = prompt::resume_after_drop(&dropped.why);
3431            retry.timeout = retry_budget(job.timeout, true);
3432            retry.stem = format!("{}-resume", job.stem);
3433            let cache = self.state.config.cache_dir();
3434            let ctx = WaveCtx {
3435                carry_seats: false,
3436                pauses: Some((&self.pause, &self.interrupt)),
3437                run: run_id,
3438                node: "implement",
3439                prompts,
3440                cache: cache.as_deref(),
3441                round: None,
3442            };
3443            let (resumed_seat, resumed) =
3444                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3445            *seat = resumed_seat;
3446            *out = resumed;
3447        }
3448    }
3449
3450    /// Fall an implement seat through to the next untried agent in the
3451    /// implementer roster when it lost to quota — or, since the handover was
3452    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3453    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3454    /// quota path itself being unchanged — instead of leaving the
3455    /// seat's loss final the moment one agent's account runs dry.
3456    ///
3457    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3458    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3459    /// truncates to the single slot rotation picked, so a solo task whose one
3460    /// implementer hits quota mid-run used to have nothing else to try. This
3461    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3462    /// unrotated roster — which is the only place the *other* candidates in
3463    /// the machine's roster still exist once `implementers` has been cut down
3464    /// to size.
3465    ///
3466    /// Walks forward from just past the seat's own original position in the
3467    /// roster, never wrapping back to the front: a later candidate slot (say
3468    /// `beta`, the roster's second entry) must fall through to the *next*
3469    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3470    /// almost certainly a different candidate's own agent already — and once
3471    /// the roster's tail is exhausted there is nothing left to fall through
3472    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3473    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3474    /// let this retry that id forever. The loop keeps falling through until
3475    /// an attempt lands something other than `Quota` or the roster's tail
3476    /// runs out of untried ids, at which point the seat is left exactly as
3477    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3478    /// `QuotaLoss` recorded, the candidate failed/empty.
3479    ///
3480    /// `sent` is taken mutably and updated with the fallback agent's spec:
3481    /// `resume_unconfirmed_commands`, which runs after this and also reads
3482    /// `sent`, must see whichever agent actually ended up answering the seat
3483    /// — reading the stale, original spec there would check session
3484    /// eligibility against the wrong CLI and could hand a fallback agent's
3485    /// session id to the agent that just lost the seat to quota.
3486    ///
3487    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3488    /// — `self.seat` only reuses state when the agent id is unchanged, so
3489    /// handing it a different id already gets this for free. Reusing the old
3490    /// seat would resume a different CLI's session as if it were a
3491    /// continuation of this one.
3492    ///
3493    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3494    /// a quota loss cuts an agent off mid-turn, so anything already in the
3495    /// tree is unfinished work, not a completed candidate a re-ask would pay
3496    /// for twice. A dirty tree is rescued into a commit first (the same
3497    /// neutral-identity rescue `implement`'s own outcome loop gives every
3498    /// candidate) so the next agent starts clean.
3499    ///
3500    /// The new agent gets the implementer's full prompt and full
3501    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3502    /// it has no session and no context, and is implementing the task from
3503    /// nothing, unlike a resumed drop which is only restating work already
3504    /// done.
3505    ///
3506    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3507    /// `implement` event, never into `self.state.quota` — that is what
3508    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3509    /// go unspent, and a seat that ultimately recovered on its second or
3510    /// third agent is not the stalled panel that check exists to catch. Only
3511    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3512    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3513    /// outcome loop already has — this helper never pushes to it itself.
3514    async fn resume_seat_handovers(
3515        &mut self,
3516        results: &mut [(usize, SeatState, AgentOutcome)],
3517        sent: &mut [SeatJob],
3518        prompts: &Prompts,
3519        run_id: &str,
3520    ) -> Result<()> {
3521        let instruction = seeded_instruction(&self.state);
3522        let language = self.state.config.graph.language.clone();
3523        let github_language = self.state.config.graph.github_language.clone();
3524        let brief = self
3525            .state
3526            .advice
3527            .as_ref()
3528            .and_then(|a| a.synthesis.as_deref())
3529            .map(str::to_owned);
3530        let attachments = self.state.attachments.clone();
3531        for (wi, seat, out) in results.iter_mut() {
3532            // Who holds the other candidate seats of this wave right now
3533            // (earlier handovers already written back to `sent`).
3534            let others: BTreeSet<String> = sent
3535                .iter()
3536                .enumerate()
3537                .filter(|(j, _)| j != wi)
3538                .map(|(_, j)| j.spec.id.clone())
3539                .collect();
3540            let Some(job) = sent.get_mut(*wi) else {
3541                continue;
3542            };
3543            // Where the seat's own original agent sits in the roster — the
3544            // fallback walk starts just past here, never at the front, so a
3545            // later candidate slot's quota loss does not fall back onto an
3546            // earlier slot's own agent.
3547            let start = self
3548                .roles
3549                .implementer_roster
3550                .iter()
3551                .position(|s| s.id == job.spec.id)
3552                .unwrap_or(0);
3553            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3554            let mut fallback_attempt = 0usize;
3555            let mut prev: Option<FailClass> = None;
3556            while let Some(cur) = FailClass::of(&*out) {
3557                if !should_hand_over(prev.as_ref(), &cur) {
3558                    break;
3559                }
3560                let Some(next) =
3561                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3562                        .cloned()
3563                else {
3564                    break;
3565                };
3566                tried.insert(next.id.clone());
3567                fallback_attempt += 1;
3568
3569                let coordination = self.coordinate(&job.cwd).await?;
3570                if let Ok(r) = git::rescue_commit(
3571                    &job.cwd,
3572                    &format!(
3573                        "magi: candidate {} (uncommitted work before {} fallback)",
3574                        seat.key,
3575                        if cur == FailClass::Quota {
3576                            "quota"
3577                        } else {
3578                            "handover"
3579                        }
3580                    ),
3581                )
3582                .await
3583                {
3584                    self.state.note_withheld("implement", &r.withheld);
3585                }
3586                drop(coordination);
3587
3588                record_handover(
3589                    &mut self.state,
3590                    "implement",
3591                    &seat.key,
3592                    &seat.agent,
3593                    &next.id,
3594                    &cur,
3595                    &fail_reason(&*out),
3596                );
3597                prev = Some(cur.clone());
3598
3599                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3600                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3601                // Kept in sync on `sent` itself, not just the local retry: a
3602                // later helper (`resume_unconfirmed_commands`) reads `sent`
3603                // after this one returns and must see whichever agent is now
3604                // occupying the seat, not the one that just quota'd out —
3605                // otherwise it would judge session/continuation eligibility
3606                // by the wrong CLI and could resend a fallback's session id
3607                // to the agent that lost it the seat in the first place.
3608                job.spec = next.clone();
3609                let mut retry = job.clone();
3610                retry.seat = new_seat;
3611                retry.prompt = prompt::implement_in(
3612                    &instruction,
3613                    &job.cwd.to_string_lossy(),
3614                    &language,
3615                    &github_language,
3616                    brief.as_deref(),
3617                    &attachments,
3618                );
3619                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3620                let cache = self.state.config.cache_dir();
3621                let ctx = WaveCtx {
3622                    carry_seats: false,
3623                    pauses: Some((&self.pause, &self.interrupt)),
3624                    run: run_id,
3625                    node: "implement",
3626                    prompts,
3627                    cache: cache.as_deref(),
3628                    round: None,
3629                };
3630                let (fallback_seat, fallback_out) = run_one(
3631                    retry,
3632                    Arc::clone(&self.sem),
3633                    &ctx,
3634                    &mut self.state,
3635                    fallback_attempt,
3636                )
3637                .await;
3638                *seat = fallback_seat;
3639                *out = fallback_out;
3640            }
3641        }
3642        Ok(())
3643    }
3644
3645    /// Ask an implement seat's own CLI to confirm what it started, once, when
3646    /// its reply reported a command whose completion status it never
3647    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3648    /// that does and does not mean.
3649    ///
3650    /// The completion contract this task asks for, extended to `implement`
3651    /// with the same signal `continue_fix_report` reads for the fixer,
3652    /// rather than a keyword search over the reply or a hard requirement on
3653    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3654    /// a candidate's CLI turn ended cleanly while a test run it had started
3655    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3656    /// named in it at all is untouched by this: `commands` is empty, so
3657    /// there is nothing to be unconfirmed.
3658    ///
3659    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3660    /// this is not about recovering edits that might already be on disk, it
3661    /// is about a result the seat itself never vouched for, which resuming
3662    /// asks for regardless of what the tree already holds. Bounded to one
3663    /// attempt for the same reason `resume_undelivered` is — this is the
3664    /// most expensive node in the graph — and a seat that still cannot
3665    /// confirm on that attempt is left as whatever its (possibly still
3666    /// unconfirmed) reply says; this does not invent a new "failed" reason
3667    /// for a candidate that otherwise produced a real, committed change.
3668    async fn resume_unconfirmed_commands(
3669        &mut self,
3670        results: &mut [(usize, SeatState, AgentOutcome)],
3671        sent: &[SeatJob],
3672        prompts: &Prompts,
3673        run_id: &str,
3674    ) {
3675        for (wi, seat, out) in results.iter_mut() {
3676            let AgentOutcome::Ok(o) = &*out else {
3677                continue;
3678            };
3679            if !has_unconfirmed_command(&o.commands) {
3680                continue;
3681            }
3682            let Some(job) = sent.get(*wi) else { continue };
3683            if !has_context(&job.spec, seat, job.sessions) {
3684                self.state.event(
3685                    "implement",
3686                    format!(
3687                        "{}: the reply named a command whose own CLI never confirmed the exit \
3688                         status of, but there is no session left to resume",
3689                        seat.key
3690                    ),
3691                );
3692                continue;
3693            }
3694            self.state.event(
3695                "implement",
3696                format!(
3697                    "{}: the reply named a command whose own CLI never confirmed the exit \
3698                     status of; resuming the conversation",
3699                    seat.key
3700                ),
3701            );
3702            let mut retry = job.clone();
3703            retry.seat = seat.clone();
3704            retry.prompt = prompt::resume_incomplete(
3705                "a command in your last reply had no confirmed exit status",
3706            );
3707            retry.timeout = retry_budget(job.timeout, true);
3708            retry.stem = format!("{}-confirm", job.stem);
3709            let cache = self.state.config.cache_dir();
3710            let ctx = WaveCtx {
3711                carry_seats: false,
3712                pauses: Some((&self.pause, &self.interrupt)),
3713                run: run_id,
3714                node: "implement",
3715                prompts,
3716                cache: cache.as_deref(),
3717                round: None,
3718            };
3719            let (resumed_seat, resumed) =
3720                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3721            *seat = resumed_seat;
3722            *out = resumed;
3723        }
3724    }
3725
3726    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3727    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3728    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3729    /// that motivated this.
3730    ///
3731    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3732    /// own nudge loop already covers for judge/review/vote seats, and not a
3733    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3734    /// implement seats: here the CLI turn genuinely finished while the node's
3735    /// own work — the fixer's account of what it did — had not. Gated purely
3736    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3737    /// reply, never on any wording in it, so a fixer whose valid, first-try
3738    /// `FixReport` happens to mention having waited on a background test is
3739    /// never resumed — the `Ok(report)` branch at the call site returns
3740    /// before this is ever invoked.
3741    ///
3742    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3743    /// attempt ([`retry_budget`]), nothing attempted once the session is
3744    /// gone, and a quota hit ends the loop immediately rather than retrying a
3745    /// rate limit that fails the same way again.
3746    async fn continue_fix_report(
3747        &mut self,
3748        mut seat: SeatState,
3749        parse_err: String,
3750        job: &SeatJob,
3751        prompts: &Prompts,
3752        run_id: &str,
3753        round: usize,
3754    ) -> (
3755        SeatState,
3756        Option<FixReport>,
3757        Option<String>,
3758        ContinuationRecord,
3759    ) {
3760        let mut last_err = parse_err;
3761        let mut cumulative_wait_ms = 0u64;
3762        let mut attempts = 0usize;
3763        loop {
3764            if !has_context(&job.spec, &seat, job.sessions) {
3765                self.state.event(
3766                    "fix",
3767                    format!(
3768                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3769                         session left to resume into"
3770                    ),
3771                );
3772                let outcome = if attempts == 0 {
3773                    ContinuationOutcome::NoSession
3774                } else {
3775                    ContinuationOutcome::Exhausted
3776                };
3777                return (
3778                    seat,
3779                    None,
3780                    Some(format!("unparsable fix report: {last_err}")),
3781                    ContinuationRecord {
3782                        attempts,
3783                        cumulative_wait_ms,
3784                        outcome,
3785                    },
3786                );
3787            }
3788            if attempts >= MAX_FIX_CONTINUATIONS {
3789                self.state.event(
3790                    "fix",
3791                    format!(
3792                        "round {round}: fixer's reply still had no adoption report after \
3793                         {attempts} continuation(s) ({last_err}); giving up"
3794                    ),
3795                );
3796                return (
3797                    seat,
3798                    None,
3799                    Some(format!(
3800                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3801                    )),
3802                    ContinuationRecord {
3803                        attempts,
3804                        cumulative_wait_ms,
3805                        outcome: ContinuationOutcome::Exhausted,
3806                    },
3807                );
3808            }
3809            attempts += 1;
3810            self.state.event(
3811                "fix",
3812                format!(
3813                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3814                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3815                ),
3816            );
3817            let mut retry = job.clone();
3818            retry.seat = seat.clone();
3819            retry.prompt = prompt::resume_incomplete(&last_err);
3820            retry.timeout = retry_budget(job.timeout, true);
3821            retry.stem = format!("{}-continue{attempts}", job.stem);
3822            let cache = self.state.config.cache_dir();
3823            let ctx = WaveCtx {
3824                carry_seats: false,
3825                pauses: Some((&self.pause, &self.interrupt)),
3826                run: run_id,
3827                node: "fix",
3828                prompts,
3829                cache: cache.as_deref(),
3830                round: Some(round),
3831            };
3832            let (resumed_seat, resumed_out) = run_one(
3833                retry,
3834                Arc::clone(&self.sem),
3835                &ctx,
3836                &mut self.state,
3837                attempts,
3838            )
3839            .await;
3840            seat = resumed_seat;
3841            match resumed_out {
3842                AgentOutcome::Ok(o) => {
3843                    cumulative_wait_ms += o.duration_ms;
3844                    match verdict::extract_json::<FixReport>(&o.text) {
3845                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3846                            self.state.event(
3847                                "fix",
3848                                format!(
3849                                    "round {round}: fixer's adoption report recovered after \
3850                                     {attempts} continuation(s)"
3851                                ),
3852                            );
3853                            return (
3854                                seat,
3855                                Some(report),
3856                                None,
3857                                ContinuationRecord {
3858                                    attempts,
3859                                    cumulative_wait_ms,
3860                                    outcome: ContinuationOutcome::Resumed,
3861                                },
3862                            );
3863                        }
3864                        // The report parsed, but this same reply's own
3865                        // CommandEvidence — the identical record `state.jobs`
3866                        // renders — names a command whose CLI never
3867                        // confirmed an exit status. Read together, that is
3868                        // not a resolved answer: keep nudging rather than
3869                        // accept a report standing next to a command the
3870                        // seat's own CLI cannot vouch for.
3871                        Ok(_) => {
3872                            last_err = "the reply parsed, but it reported a command whose own CLI \
3873                                 never confirmed an exit status"
3874                                .to_owned();
3875                        }
3876                        Err(e) => last_err = e.to_string(),
3877                    }
3878                }
3879                AgentOutcome::Quota(o) => {
3880                    cumulative_wait_ms += o.duration_ms;
3881                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3882                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3883                    self.state.quota.push(loss);
3884                    self.state.event(
3885                        "fix",
3886                        format!(
3887                            "round {round}: continuation rate limited (quota); not retrying now"
3888                        ),
3889                    );
3890                    return (
3891                        seat,
3892                        None,
3893                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3894                        ContinuationRecord {
3895                            attempts,
3896                            cumulative_wait_ms,
3897                            outcome: ContinuationOutcome::QuotaLost,
3898                        },
3899                    );
3900                }
3901                AgentOutcome::Dropped(o) => {
3902                    cumulative_wait_ms += o.duration_ms;
3903                    let why = o
3904                        .dropped
3905                        .as_ref()
3906                        .map(|d| d.why.as_str())
3907                        .unwrap_or("the CLI ended the stream without delivering its answer");
3908                    last_err = format!("the CLI dropped the stream ({why})");
3909                }
3910                AgentOutcome::Failed(e) => last_err = e,
3911                // The continuation was never started; the report stays lost
3912                // and the caller records that, without any handover.
3913                AgentOutcome::ResourceBlocked(e) => {
3914                    return (
3915                        seat,
3916                        None,
3917                        Some(format!("the continuation was not started: {e}")),
3918                        ContinuationRecord {
3919                            attempts: attempts.saturating_sub(1),
3920                            cumulative_wait_ms,
3921                            outcome: ContinuationOutcome::NoSession,
3922                        },
3923                    );
3924                }
3925            }
3926        }
3927    }
3928
3929    fn after_implement(&mut self) -> Result<()> {
3930        if !self.state.candidate_preservation.is_empty() {
3931            bail!("candidate preservation incomplete; judging is unsafe");
3932        }
3933        // Scan every candidate patch once the set is complete.
3934        if self.state.leaks.is_empty() {
3935            let cfg = self.state.config.blind.clone();
3936            let mut leaks = Vec::new();
3937            for c in &self.state.candidates {
3938                let Some(patch) =
3939                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3940                else {
3941                    continue;
3942                };
3943                leaks.extend(blind::scan(
3944                    &format!("candidate {} patch", c.label),
3945                    &patch,
3946                    &cfg.vendor_tokens,
3947                ));
3948            }
3949            if !leaks.is_empty() {
3950                let summary = leaks
3951                    .iter()
3952                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3953                    .collect::<Vec<_>>()
3954                    .join(", ");
3955                match cfg.on_leak {
3956                    LeakPolicy::Fail => {
3957                        self.state.status = RunStatus::Failed;
3958                        self.state
3959                            .event("blind", format!("vendor text in a patch: {summary}"));
3960                        self.state.leaks = leaks;
3961                        self.state.save()?;
3962                        self.settle_questions();
3963                        bail!(
3964                            "blind.on_leak = \"fail\" and vendor text reached a \
3965                             judged patch: {summary}"
3966                        );
3967                    }
3968                    LeakPolicy::Redact => self.state.event(
3969                        "blind",
3970                        format!("redacting vendor text for judging: {summary}"),
3971                    ),
3972                    LeakPolicy::Warn => self.state.event(
3973                        "blind",
3974                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3975                    ),
3976                }
3977                self.state.leaks = leaks;
3978            }
3979        }
3980
3981        if self.state.viable().is_empty() {
3982            if self.state.all_candidates_verified_noop() {
3983                // Every candidate agreed, with evidence the adoption guard
3984                // accepted, that nothing belongs in this worktree. That is
3985                // not the same fact as a candidate that simply failed to
3986                // write anything, and settling it as an ordinary `Failed`
3987                // (see `SCHEMA`'s doc for schema 10) is what let two of
3988                // task 391f's attempts burn a retry each re-discovering the
3989                // same already-landed fix. Terminal either way, so `judge`
3990                // must never run over an empty candidate set — unlike the
3991                // `Failed` branch below this returns `Ok`, not an error:
3992                // nothing here failed.
3993                self.state.status = RunStatus::VerifiedNoop;
3994                self.state.save()?;
3995                self.settle_questions();
3996                return Ok(());
3997            }
3998            self.state.status = RunStatus::Failed;
3999            self.state.save()?;
4000            self.settle_questions();
4001            bail!("no candidate produced a change; nothing to judge");
4002        }
4003        self.state.status = RunStatus::Judging;
4004        self.state.save()?;
4005        Ok(())
4006    }
4007
4008    // --------------------------------------------------------------- judge
4009
4010    async fn judge(&mut self) -> Result<()> {
4011        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4012        // agent files with `magi task add` name the run that paid for it. The
4013        // prompt overlay is cloned alongside it because the waves borrow it
4014        // while `self` is mutably borrowed by the node's own bookkeeping.
4015        let run_id = self.state.id.clone();
4016        let prompts = self.state.config.prompts.clone();
4017        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4018            return Ok(());
4019        }
4020        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4021        if viable.len() == 1 {
4022            // Recorded so this is a one-time event: `judgements` stays empty
4023            // either way, which without this flag is indistinguishable from
4024            // "not yet judged" on the next reentry — and status is left
4025            // untouched, so a later node's conclusion (e.g. `Blocked` after
4026            // the review budget ran out) survives a resume instead of being
4027            // clobbered back to `Judging` by this node running again.
4028            self.state.judge_skipped = true;
4029            self.state.event(
4030                "judge",
4031                format!(
4032                    "only candidate {} produced a change; judging skipped",
4033                    viable[0].label
4034                ),
4035            );
4036            self.state.save()?;
4037            return Ok(());
4038        }
4039        self.state.status = RunStatus::Judging;
4040
4041        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4042        let language = self.state.config.graph.language.clone();
4043        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4044        let sessions = self.state.config.graph.sessions;
4045        let artifacts = agent::artifacts_dir(&self.state.dir());
4046        let root = self.state.worktree_root();
4047        let base_short = short(&self.state.base_commit);
4048
4049        let mut jobs = Vec::new();
4050        let mut orders = Vec::new();
4051        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4052            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4053            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4054            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4055            let seat_key = format!("judge-{}", j + 1);
4056            let seat = self.seat(&seat_key, &spec.id);
4057            jobs.push(SeatJob {
4058                prompt: prompt::judge(
4059                    &self.state.instruction,
4060                    &views,
4061                    self.roles.judges.len(),
4062                    &base_short,
4063                    &language,
4064                ),
4065                spec,
4066                seat,
4067                cwd: root.join(format!("judge-{}", j + 1)),
4068                timeout,
4069                allow_write: false,
4070                sessions,
4071                artifacts: artifacts.clone(),
4072                stem: format!("judge-{}", j + 1),
4073                handover: None,
4074            });
4075        }
4076
4077        self.state.event(
4078            "judge",
4079            format!(
4080                "{} judges ranking {} candidates blind",
4081                jobs.len(),
4082                viable.len()
4083            ),
4084        );
4085        let labels_for_check = labels.clone();
4086        let mut quota_losses = Vec::new();
4087        let cache = self.state.config.cache_dir();
4088        let ctx = WaveCtx {
4089            carry_seats: false,
4090            pauses: Some((&self.pause, &self.interrupt)),
4091            run: &run_id,
4092            node: "judge",
4093            prompts: &prompts,
4094            cache: cache.as_deref(),
4095            round: None,
4096        };
4097        let results = ask_json_wave::<Ranking>(
4098            jobs,
4099            Arc::clone(&self.sem),
4100            self.state.config.graph.retries,
4101            &self.roles.judge_roster,
4102            &ctx,
4103            &mut quota_losses,
4104            &mut self.state,
4105            &move |r: &Ranking| r.validate(&labels_for_check),
4106        )
4107        .await;
4108        if self.state.resource_block.is_some() {
4109            return self.stop_for_resource();
4110        }
4111        self.state.quota.extend(quota_losses);
4112
4113        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4114            let agent_id = seat.agent.clone();
4115            self.state.seats.insert(seat.key.clone(), seat);
4116            let mut record = Judgement {
4117                judge: j + 1,
4118                seat: format!("judge-{}", j + 1),
4119                agent: agent_id,
4120                ranking: Vec::new(),
4121                reasons: BTreeMap::new(),
4122                confidence: None,
4123                strengths: BTreeMap::new(),
4124                order: orders[j].clone(),
4125                failed: None,
4126                duration_ms: 0,
4127            };
4128            match res {
4129                Ok((ranking, out)) => {
4130                    record.ranking = ranking.normalized();
4131                    record.reasons = ranking.reasons;
4132                    record.strengths = ranking.strengths;
4133                    record.confidence = ranking.confidence;
4134                    record.duration_ms = out.duration_ms;
4135                    self.state.event(
4136                        "judge",
4137                        format!(
4138                            "judge {} ranked {}",
4139                            j + 1,
4140                            record.ranking.iter().collect::<String>()
4141                        ),
4142                    );
4143                }
4144                Err(e) => {
4145                    record.failed = Some(e.to_string());
4146                    self.state
4147                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4148                }
4149            }
4150            self.state.judgements.push(record);
4151            self.state.save()?;
4152        }
4153        Ok(())
4154    }
4155
4156    // ---------------------------------------------------------- deliberate
4157
4158    async fn deliberate(&mut self) -> Result<()> {
4159        if self.hold_base_sync().await? {
4160            return Ok(());
4161        }
4162        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4163        // agent files with `magi task add` name the run that paid for it. The
4164        // prompt overlay is cloned alongside it because the waves borrow it
4165        // while `self` is mutably borrowed by the node's own bookkeeping.
4166        let run_id = self.state.id.clone();
4167        let prompts = self.state.config.prompts.clone();
4168        let rounds = self.state.config.graph.deliberate_rounds;
4169        // Resume unfinished rounds after an outage, but never reopen a tally.
4170        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4171            return Ok(());
4172        }
4173        let tops: Vec<char> = self
4174            .state
4175            .judgements
4176            .iter()
4177            .filter_map(|j| j.ranking.first().copied())
4178            .collect();
4179        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4180            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4181                self.state.event(
4182                    "deliberate",
4183                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4184                );
4185            }
4186            self.state.status = RunStatus::Voting;
4187            self.state.save()?;
4188            return Ok(());
4189        }
4190
4191        self.state.status = RunStatus::Deliberating;
4192        self.state.event(
4193            "deliberate",
4194            format!(
4195                "split: first choices were {} — opening {rounds} round(s)",
4196                tops.iter().collect::<String>()
4197            ),
4198        );
4199
4200        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4201        let language = self.state.config.graph.language.clone();
4202        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4203        let sessions = self.state.config.graph.sessions;
4204        let artifacts = agent::artifacts_dir(&self.state.dir());
4205        let root = self.state.worktree_root();
4206        let base_short = short(&self.state.base_commit);
4207
4208        // Judges argue in sequence so that a turn can answer the one before it;
4209        // that is the difference between deliberation and three parallel
4210        // monologues.
4211        for round in self.state.deliberation.len() + 1..=rounds {
4212            let mut turns: Vec<DeliberationTurn> = Vec::new();
4213            let mut round_losses = Vec::new();
4214            let mut asked = 0;
4215            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4216                if self.state.judgements[j].failed.is_some() {
4217                    continue;
4218                }
4219                let seat_key = format!("judge-{}", j + 1);
4220                let spec = self.occupant(&seat_key, spec);
4221                let mut seat = self.seat(&seat_key, &spec.id);
4222                let transcript = self.transcript(&turns, j);
4223                let build = |context: Option<&str>| {
4224                    prompt::deliberate(
4225                        &self.state.instruction,
4226                        context,
4227                        &transcript,
4228                        round,
4229                        rounds,
4230                        &language,
4231                    )
4232                };
4233                let block = self.candidate_block(&viable, &base_short);
4234                let full = build(Some(&block));
4235                let text = if has_context(&spec, &seat, sessions) {
4236                    build(None)
4237                } else {
4238                    full.clone()
4239                };
4240                let job = SeatJob {
4241                    spec,
4242                    seat: seat.clone(),
4243                    prompt: text,
4244                    cwd: root.join(format!("judge-{}", j + 1)),
4245                    timeout,
4246                    allow_write: false,
4247                    sessions,
4248                    artifacts: artifacts.clone(),
4249                    stem: format!("delib-{round}-judge-{}", j + 1),
4250                    handover: Some(full),
4251                };
4252                let cache = self.state.config.cache_dir();
4253                let ctx = WaveCtx {
4254                    carry_seats: false,
4255                    pauses: Some((&self.pause, &self.interrupt)),
4256                    run: &run_id,
4257                    node: "deliberate",
4258                    prompts: &prompts,
4259                    cache: cache.as_deref(),
4260                    round: None,
4261                };
4262                // A turn is never nudged (`retries` 0): a failed seat is
4263                // handed to the next roster agent, which gets the full
4264                // context. An empty answer is a turn, not a failure.
4265                let mut losses = Vec::new();
4266                asked += 1;
4267                let mut results = ask_wave_with::<String>(
4268                    vec![job],
4269                    Arc::clone(&self.sem),
4270                    0,
4271                    &self.roles.judge_roster,
4272                    &ctx,
4273                    &mut losses,
4274                    &mut self.state,
4275                    &|text: &str| {
4276                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4277                    },
4278                )
4279                .await;
4280                if self.state.resource_block.is_some() {
4281                    return self.stop_for_resource();
4282                }
4283                round_losses.extend(losses.iter().cloned());
4284                self.state.quota.extend(losses);
4285                let (updated, res, _) = results.pop().expect("one job in, one result out");
4286                seat = updated;
4287                let agent_id = seat.agent.clone();
4288                self.state.seats.insert(seat.key.clone(), seat);
4289                let body = match res {
4290                    Ok((body, _)) => body,
4291                    // Skip the seat; a CLI's raw error JSON is never read as
4292                    // this judge's position.
4293                    Err(e) => {
4294                        self.state
4295                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4296                        continue;
4297                    }
4298                };
4299                let tentative = verdict::extract_json::<Position>(&body)
4300                    .ok()
4301                    .and_then(|p| p.tentative)
4302                    .and_then(|s| s.trim().chars().next())
4303                    .map(|c| c.to_ascii_uppercase());
4304                self.state.event(
4305                    "deliberate",
4306                    format!(
4307                        "round {round}: judge {} now favours {}",
4308                        j + 1,
4309                        tentative.map_or("—".to_owned(), |c| c.to_string())
4310                    ),
4311                );
4312                turns.push(DeliberationTurn {
4313                    judge: j + 1,
4314                    agent: agent_id,
4315                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4316                    tentative,
4317                });
4318            }
4319            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4320                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4321                return self.stop_for_resource();
4322            }
4323            self.state
4324                .deliberation
4325                .push(DeliberationRound { round, turns });
4326            self.state.save()?;
4327        }
4328
4329        self.state.status = RunStatus::Voting;
4330        self.state.save()?;
4331        Ok(())
4332    }
4333
4334    // ---------------------------------------------------------------- vote
4335
4336    async fn vote(&mut self) -> Result<()> {
4337        if self.hold_base_sync().await? {
4338            return Ok(());
4339        }
4340        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4341        // agent files with `magi task add` name the run that paid for it. The
4342        // prompt overlay is cloned alongside it because the waves borrow it
4343        // while `self` is mutably borrowed by the node's own bookkeeping.
4344        let run_id = self.state.id.clone();
4345        let prompts = self.state.config.prompts.clone();
4346        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4347            return Ok(());
4348        }
4349        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4350        if viable.len() == 1 {
4351            return Ok(());
4352        }
4353        self.state.status = RunStatus::Voting;
4354
4355        let language = self.state.config.graph.language.clone();
4356        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4357        let sessions = self.state.config.graph.sessions;
4358        let artifacts = agent::artifacts_dir(&self.state.dir());
4359        let root = self.state.worktree_root();
4360        let base_short = short(&self.state.base_commit);
4361        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4362
4363        let mut jobs = Vec::new();
4364        let mut seats_at = Vec::new();
4365        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4366            if self
4367                .state
4368                .judgements
4369                .get(j)
4370                .is_some_and(|r| r.failed.is_some())
4371            {
4372                continue;
4373            }
4374            let seat_key = format!("judge-{}", j + 1);
4375            let spec = self.occupant(&seat_key, spec);
4376            let seat = self.seat(&seat_key, &spec.id);
4377            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4378            let text = if has_context(&spec, &seat, sessions) {
4379                prompt::final_vote(&viable, &language)
4380            } else {
4381                full.clone()
4382            };
4383            jobs.push(SeatJob {
4384                spec,
4385                seat,
4386                prompt: text,
4387                cwd: root.join(format!("judge-{}", j + 1)),
4388                timeout,
4389                allow_write: false,
4390                sessions,
4391                artifacts: artifacts.clone(),
4392                stem: format!("vote-judge-{}", j + 1),
4393                handover: Some(full),
4394            });
4395            seats_at.push(j);
4396        }
4397
4398        self.state.event(
4399            "vote",
4400            format!(
4401                "collecting {} final votes one by one, privately",
4402                jobs.len()
4403            ),
4404        );
4405        let allowed = viable.clone();
4406        let mut quota_losses = Vec::new();
4407        let cache = self.state.config.cache_dir();
4408        let ctx = WaveCtx {
4409            carry_seats: false,
4410            pauses: Some((&self.pause, &self.interrupt)),
4411            run: &run_id,
4412            node: "vote",
4413            prompts: &prompts,
4414            cache: cache.as_deref(),
4415            round: None,
4416        };
4417        let results = ask_json_wave::<FinalVote>(
4418            jobs,
4419            Arc::clone(&self.sem),
4420            self.state.config.graph.retries,
4421            &self.roles.judge_roster,
4422            &ctx,
4423            &mut quota_losses,
4424            &mut self.state,
4425            &move |v: &FinalVote| match v.label() {
4426                Some(c) if allowed.contains(&c) => Ok(()),
4427                other => bail!("vote {other:?} is not one of {allowed:?}"),
4428            },
4429        )
4430        .await;
4431        if self.state.resource_block.is_some() {
4432            return self.stop_for_resource();
4433        }
4434        self.state.quota.extend(quota_losses);
4435
4436        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4437            let agent_id = seat.agent.clone();
4438            self.state.seats.insert(seat.key.clone(), seat);
4439            let initial = self
4440                .state
4441                .judgements
4442                .get(j)
4443                .and_then(|r| r.ranking.first().copied());
4444            let mut record = VoteRecord {
4445                judge: j + 1,
4446                agent: agent_id,
4447                vote: None,
4448                reason: String::new(),
4449                changed: false,
4450            };
4451            match res {
4452                Ok((v, _)) => {
4453                    record.vote = v.label();
4454                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4455                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4456                    self.state.event(
4457                        "vote",
4458                        format!(
4459                            "judge {} voted {}{}",
4460                            j + 1,
4461                            record.vote.unwrap_or('?'),
4462                            if record.changed { " (changed)" } else { "" }
4463                        ),
4464                    );
4465                }
4466                Err(e) => {
4467                    self.state
4468                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4469                }
4470            }
4471            self.state.votes.push(record);
4472            self.state.save()?;
4473        }
4474        Ok(())
4475    }
4476
4477    // --------------------------------------------------------------- tally
4478
4479    fn tally(&mut self) -> Result<()> {
4480        if self.state.tally.is_some() {
4481            return Ok(());
4482        }
4483        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4484        let tops: Vec<char> = self
4485            .state
4486            .judgements
4487            .iter()
4488            .filter_map(|j| j.ranking.first().copied())
4489            .collect();
4490        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4491
4492        // A judge whose private vote failed still counted once, in the initial
4493        // ranking; using it beats discarding a whole seat.
4494        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4495        let mut cast: Vec<char> = Vec::new();
4496        for (i, j) in self.state.judgements.iter().enumerate() {
4497            let vote = self
4498                .state
4499                .votes
4500                .iter()
4501                .find(|v| v.judge == i + 1)
4502                .and_then(|v| v.vote)
4503                .or_else(|| j.ranking.first().copied());
4504            if let Some(v) = vote {
4505                *first_choice.entry(v).or_insert(0) += 1;
4506                cast.push(v);
4507            }
4508        }
4509
4510        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4511        for j in &self.state.judgements {
4512            let n = j.ranking.len();
4513            for (pos, label) in j.ranking.iter().enumerate() {
4514                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4515            }
4516        }
4517
4518        let best = first_choice.values().copied().max().unwrap_or(0);
4519        let mut leaders: Vec<char> = first_choice
4520            .iter()
4521            .filter(|(_, v)| **v == best)
4522            .map(|(k, _)| *k)
4523            .collect();
4524        let mut tie_break = None;
4525        if leaders.len() > 1 {
4526            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4527            let borda_leaders: Vec<char> = leaders
4528                .iter()
4529                .copied()
4530                .filter(|l| borda[l] == top_borda)
4531                .collect();
4532            tie_break = Some(if borda_leaders.len() == 1 {
4533                format!(
4534                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4535                    leaders.len()
4536                )
4537            } else {
4538                format!(
4539                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4540                    leaders.len()
4541                )
4542            });
4543            leaders = borda_leaders;
4544            leaders.sort_unstable();
4545        }
4546        let winner = *leaders
4547            .first()
4548            .or(viable.first())
4549            .context("no candidate to declare a winner from")?;
4550
4551        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4552        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4553        let deliberated = !self.state.deliberation.is_empty();
4554
4555        // Whose verdict is this? A rate-limited seat is absent even if it
4556        // ranked before the limit hit, so presence is measured against the
4557        // recorded losses, not just "did a ranking ever appear".
4558        let quota_seats: std::collections::BTreeSet<&str> =
4559            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4560        let mut present = 0usize;
4561        for (i, j) in self.state.judgements.iter().enumerate() {
4562            if quota_seats.contains(j.seat.as_str()) {
4563                continue;
4564            }
4565            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4566            let voted = self
4567                .state
4568                .votes
4569                .iter()
4570                .any(|v| v.judge == i + 1 && v.vote.is_some());
4571            if ranked || voted {
4572                present += 1;
4573            }
4574        }
4575        // Strict majority of the configured panel. A bare majority is real
4576        // signal we can act on, while a minority verdict must never stand in
4577        // for a healthy one. A one-candidate run needs no panel at all, and
4578        // `judges` stays `0` rather than the roster size a panel that never
4579        // sat would otherwise be credited with.
4580        let needs_quorum = viable.len() > 1;
4581        let judges_total = if needs_quorum {
4582            self.roles.judges.len()
4583        } else {
4584            0
4585        };
4586        let quorum = if needs_quorum {
4587            judges_total / 2 + 1
4588        } else {
4589            0
4590        };
4591        let met_quorum = !needs_quorum || present >= quorum;
4592        let uncontested = (!needs_quorum).then(|| {
4593            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4594        });
4595
4596        self.state.event(
4597            "tally",
4598            match &uncontested {
4599                Some(reason) => format!("winner {winner} — {reason}"),
4600                None => format!(
4601                    "winner {winner} — votes {} | initial {} | {} changed | \
4602                     {present}/{judges_total} judges{}",
4603                    first_choice
4604                        .iter()
4605                        .map(|(k, v)| format!("{k}:{v}"))
4606                        .collect::<Vec<_>>()
4607                        .join(" "),
4608                    if unanimous_initial {
4609                        "unanimous"
4610                    } else {
4611                        "split"
4612                    },
4613                    changed_votes,
4614                    if met_quorum {
4615                        String::new()
4616                    } else {
4617                        format!(" — below quorum ({quorum} required)")
4618                    },
4619                ),
4620            },
4621        );
4622        if !met_quorum {
4623            self.state.event(
4624                "stall",
4625                format!(
4626                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4627                     the run stops here, resumable"
4628                ),
4629            );
4630        }
4631        self.state.tally = Some(Tally {
4632            first_choice,
4633            borda,
4634            winner,
4635            rankings: tops.len(),
4636            unanimous_initial,
4637            deliberated,
4638            changed_votes,
4639            unanimous_final,
4640            tie_break,
4641            judges: judges_total,
4642            present,
4643            quorum,
4644            met_quorum,
4645            uncontested,
4646        });
4647        self.state.status = if met_quorum {
4648            RunStatus::Reviewing
4649        } else {
4650            RunStatus::Stalled
4651        };
4652        self.state.save()?;
4653        Ok(())
4654    }
4655
4656    // ------------------------------------------------------------- recover
4657
4658    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4659    /// resumed toward completion once the transient cause clears.
4660    ///
4661    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4662    /// it toward the quorum, which is exactly the set of seats whose absence
4663    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4664    /// not depend on which node happened to hit the limit), or an ordinary
4665    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4666    /// seat is never disturbed.
4667    ///
4668    /// A seat that now answers with a usable ranking is "recovered": its
4669    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4670    /// `tally` counts it present again), and its vote re-collected. A seat that
4671    /// still fails keeps its loss and stays absent.
4672    ///
4673    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4674    /// to review/gate/merge), `false` when it is still below quorum (the run
4675    /// stays `Stalled`, still resumable for a later retry).
4676    #[allow(clippy::too_many_lines)]
4677    async fn recover_stall(&mut self) -> Result<bool> {
4678        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4679        // agent files with `magi task add` name the run that paid for it. The
4680        // prompt overlay is cloned alongside it because the waves borrow it
4681        // while `self` is mutably borrowed by the node's own bookkeeping.
4682        let run_id = self.state.id.clone();
4683        let prompts = self.state.config.prompts.clone();
4684        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4685        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4686        // is what keeps a non-quota collapse — or a quota loss recorded at the
4687        // deliberate node — from being a permanent dead-end on `--resume`.
4688        let quota_seats: BTreeSet<&str> =
4689            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4690        let absent: Vec<String> = self
4691            .state
4692            .judgements
4693            .iter()
4694            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4695            .map(|j| j.seat.clone())
4696            .collect();
4697        if absent.is_empty() {
4698            return Ok(false);
4699        }
4700        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4701        if viable.len() <= 1 {
4702            return Ok(false);
4703        }
4704        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4705        let language = self.state.config.graph.language.clone();
4706        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4707        let sessions = self.state.config.graph.sessions;
4708        let artifacts = agent::artifacts_dir(&self.state.dir());
4709        let root = self.state.worktree_root();
4710        let base_short = short(&self.state.base_commit);
4711        let candidates: Vec<Candidate> = viable.clone();
4712
4713        // Map each absent seat key to its 0-based position in `roles.judges`.
4714        let mut positions: Vec<usize> = absent
4715            .iter()
4716            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4717            .collect();
4718        if positions.is_empty() {
4719            return Ok(false);
4720        }
4721        positions.sort_unstable();
4722        positions.dedup();
4723
4724        // Re-rank the lost seats, one blind prompt each.
4725        let mut judge_jobs = Vec::new();
4726        for &j in &positions {
4727            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4728            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4729            let seat_key = format!("judge-{}", j + 1);
4730            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4731            let seat = self.seat(&seat_key, &spec.id);
4732            judge_jobs.push(SeatJob {
4733                spec,
4734                seat,
4735                prompt: prompt::judge(
4736                    &self.state.instruction,
4737                    &views,
4738                    self.roles.judges.len(),
4739                    &base_short,
4740                    &language,
4741                ),
4742                cwd: root.join(seat_key),
4743                timeout,
4744                allow_write: false,
4745                sessions,
4746                artifacts: artifacts.clone(),
4747                stem: format!("judge-{}-recover", j + 1),
4748                handover: None,
4749            });
4750        }
4751
4752        let labels_for_check = labels.clone();
4753        let mut judge_losses = Vec::new();
4754        let retries = self.state.config.graph.retries;
4755        let cache = self.state.config.cache_dir();
4756        let ctx = WaveCtx {
4757            carry_seats: false,
4758            pauses: Some((&self.pause, &self.interrupt)),
4759            run: &run_id,
4760            node: "judge",
4761            prompts: &prompts,
4762            cache: cache.as_deref(),
4763            round: None,
4764        };
4765        let results = ask_json_wave::<Ranking>(
4766            judge_jobs,
4767            Arc::clone(&self.sem),
4768            retries,
4769            &self.roles.judge_roster,
4770            &ctx,
4771            &mut judge_losses,
4772            &mut self.state,
4773            &move |r: &Ranking| r.validate(&labels_for_check),
4774        )
4775        .await;
4776        if self.state.resource_block.is_some() {
4777            self.stop_for_resource()?;
4778            return Ok(false);
4779        }
4780
4781        // Refresh the judgement of every seat that ranked again.
4782        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4783        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4784            let agent_id = seat.agent.clone();
4785            self.state.seats.insert(seat.key.clone(), seat);
4786            let record = &mut self.state.judgements[j];
4787            match res {
4788                Ok((ranking, out)) => {
4789                    record.agent = agent_id;
4790                    record.ranking = ranking.normalized();
4791                    record.reasons = ranking.reasons;
4792                    record.strengths = ranking.strengths;
4793                    record.confidence = ranking.confidence;
4794                    record.failed = None;
4795                    record.duration_ms = out.duration_ms;
4796                    recovered.insert(j);
4797                    self.state.event(
4798                        "recover",
4799                        format!("judge {} ranked again after the limit", j + 1),
4800                    );
4801                }
4802                Err(e) => {
4803                    self.state
4804                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4805                }
4806            }
4807        }
4808
4809        // Re-ask the votes of the seats that recovered a ranking.
4810        let mut vote_jobs = Vec::new();
4811        let mut vote_pos: Vec<usize> = Vec::new();
4812        for &j in &recovered {
4813            let seat_key = format!("judge-{}", j + 1);
4814            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4815            let seat = self.seat(&seat_key, &spec.id);
4816            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4817            let text = if has_context(&spec, &seat, sessions) {
4818                prompt::final_vote(&labels, &language)
4819            } else {
4820                full.clone()
4821            };
4822            vote_jobs.push(SeatJob {
4823                spec,
4824                seat,
4825                prompt: text,
4826                cwd: root.join(seat_key),
4827                timeout,
4828                allow_write: false,
4829                sessions,
4830                artifacts: artifacts.clone(),
4831                stem: format!("vote-judge-{}-recover", j + 1),
4832                handover: Some(full),
4833            });
4834            vote_pos.push(j);
4835        }
4836        let allowed = labels.clone();
4837        let mut vote_losses = Vec::new();
4838        let vote_retries = self.state.config.graph.retries;
4839        let vote_cache = self.state.config.cache_dir();
4840        let ctx = WaveCtx {
4841            carry_seats: false,
4842            pauses: Some((&self.pause, &self.interrupt)),
4843            run: &run_id,
4844            node: "vote",
4845            prompts: &prompts,
4846            cache: vote_cache.as_deref(),
4847            round: None,
4848        };
4849        let votes = ask_json_wave::<FinalVote>(
4850            vote_jobs,
4851            Arc::clone(&self.sem),
4852            vote_retries,
4853            &self.roles.judge_roster,
4854            &ctx,
4855            &mut vote_losses,
4856            &mut self.state,
4857            &move |v: &FinalVote| match v.label() {
4858                Some(c) if allowed.contains(&c) => Ok(()),
4859                other => bail!("vote {other:?} is not one of {allowed:?}"),
4860            },
4861        )
4862        .await;
4863        if self.state.resource_block.is_some() {
4864            self.stop_for_resource()?;
4865            return Ok(false);
4866        }
4867        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4868            let agent_id = seat.agent.clone();
4869            self.state.seats.insert(seat.key.clone(), seat);
4870            match res {
4871                Ok((v, _)) => {
4872                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4873                        rec.vote = v.label();
4874                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4875                    } else {
4876                        self.state.votes.push(VoteRecord {
4877                            judge: j + 1,
4878                            agent: agent_id,
4879                            vote: v.label(),
4880                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4881                            changed: false,
4882                        });
4883                    }
4884                    self.state.event(
4885                        "recover",
4886                        format!("judge {} voted again after the limit", j + 1),
4887                    );
4888                }
4889                Err(e) => {
4890                    self.state
4891                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4892                }
4893            }
4894        }
4895
4896        // A seat that ranked again is present even if its re-vote failed —
4897        // `tally` falls back to the initial ranking's first choice — so clear
4898        // its quota loss. Seats that still fail keep theirs and stay absent.
4899        let recovered_keys: BTreeSet<String> = recovered
4900            .iter()
4901            .map(|&j| format!("judge-{}", j + 1))
4902            .collect();
4903        self.state
4904            .quota
4905            .retain(|q| !recovered_keys.contains(&q.seat));
4906        // A seat that hit the limit again is a fresh loss, not the old one:
4907        // replace the stale entry so the history stays one-per-seat and the
4908        // daemon can tell this attempt's loss from a previous session's.
4909        for loss in judge_losses.into_iter().chain(vote_losses) {
4910            if recovered_keys.contains(&loss.seat) {
4911                continue;
4912            }
4913            self.state.quota.retain(|q| q.seat != loss.seat);
4914            self.state.quota.push(loss);
4915        }
4916
4917        // Recompute the verdict from the refreshed panel.
4918        self.state.tally = None;
4919        self.tally()?;
4920        Ok(self
4921            .state
4922            .tally
4923            .as_ref()
4924            .map(|t| t.met_quorum)
4925            .unwrap_or(false))
4926    }
4927
4928    // ---------------------------------------------------------------- graft
4929
4930    /// Port the strengths the judges attributed to the losing candidates onto
4931    /// the winner, once, between the tally and the review loop.
4932    ///
4933    /// Best-effort by construction: it returns nothing and fails nothing. A
4934    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4935    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4936    /// the run carries on as if the phase were off. See [`graft`] for the
4937    /// decisions and `AGENTS.md` for the invariants.
4938    ///
4939    /// The one exception is a rollback that could not be confirmed: the
4940    /// rejected commit may still be the winner's tip, so the run is blocked
4941    /// (resumable; the record stays `Running`, which makes a resume retry the
4942    /// restore) instead of reviewing, gating and merging it.
4943    async fn graft(&mut self) -> Result<()> {
4944        let Err(e) = self.graft_inner().await else {
4945            return Ok(());
4946        };
4947        if e.is::<crate::repo_lock::ResourceBlocked>() {
4948            return Err(e);
4949        }
4950        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4951            self.state.status = RunStatus::Blocked;
4952            self.state.event("graft", format!("blocked: {e:#}"));
4953            self.state.save()?;
4954            return Err(e);
4955        }
4956        // An error after the `Running` record was saved may have left the
4957        // fixer's work in the tree: the record must not stay unsettled
4958        // (a later resume would reset past the review fixes), so confirm the
4959        // restore now, or block.
4960        if let Some(rec) = self.state.graft.clone()
4961            && !rec.outcome.settled()
4962        {
4963            if let Some(winner) = self.state.winner().cloned() {
4964                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4965                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4966                        return Err(re);
4967                    }
4968                    self.state.status = RunStatus::Blocked;
4969                    self.state.event("graft", format!("blocked: {re:#}"));
4970                    self.state.save()?;
4971                    return Err(re);
4972                }
4973            }
4974            if let Some(g) = self.state.graft.as_mut() {
4975                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4976            }
4977        }
4978        self.state.event("graft", format!("skipped: {e:#}"));
4979        if let Err(e) = self.state.save() {
4980            tracing::warn!("save after graft: {e:#}");
4981        }
4982        Ok(())
4983    }
4984
4985    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4986    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4987    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4988        let _coordination = self.coordinate(wt).await?;
4989        crate::recovery::restore_repair(wt, pre)
4990            .await
4991            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4992    }
4993
4994    /// A skip is a run event, recorded once however often the node is re-entered.
4995    fn graft_skipped(&mut self, why: &str) {
4996        let message = format!("skipped: {why}");
4997        if !self
4998            .state
4999            .events
5000            .iter()
5001            .any(|e| e.node == "graft" && e.message == message)
5002        {
5003            self.state.event("graft", message);
5004        }
5005    }
5006
5007    async fn graft_inner(&mut self) -> Result<()> {
5008        // A round that was started and never finished (crash, kill): the
5009        // fixer may have left anything in the tree, so put the winner back.
5010        if let Some(rec) = self.state.graft.clone()
5011            && !rec.outcome.settled()
5012        {
5013            // Reviews exist only if the graft ended without its record being
5014            // settled (a swallowed error): the tree has moved on since, and
5015            // a reset to `pre` would discard the review fixes.
5016            if self.state.reviews.is_empty()
5017                && let Some(winner) = self.state.winner().cloned()
5018            {
5019                self.graft_restore(&winner.worktree, &rec.pre).await?;
5020            }
5021            if rec.outcome == GraftOutcome::WaitingForRestore {
5022                self.state.graft = None;
5023                self.state.save()?;
5024            } else {
5025                if let Some(g) = self.state.graft.as_mut() {
5026                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5027                }
5028                self.state.event(
5029                    "graft",
5030                    "rolled back: the previous round was interrupted; the winner is untouched",
5031                );
5032                self.state.save()?;
5033                return Ok(());
5034            }
5035        }
5036        let plan = match graft::plan(&self.state) {
5037            Ok(plan) => plan,
5038            Err(skip) => {
5039                if skip != graft::Skip::AlreadyDone {
5040                    self.graft_skipped(skip.reason());
5041                    self.state.save()?;
5042                }
5043                return Ok(());
5044            }
5045        };
5046        let Some(winner) = self.state.winner().cloned() else {
5047            return Ok(());
5048        };
5049        let wt = winner.worktree.clone();
5050        if !git::is_clean(&wt).await? {
5051            self.graft_skipped("the winner's worktree has uncommitted changes");
5052            self.state.save()?;
5053            return Ok(());
5054        }
5055        let pre = git::rev_parse(&wt, "HEAD").await?;
5056        // Saved before the fixer is called: a stop from here on is repaired
5057        // by the branch above on the next entry.
5058        self.state.graft = Some(GraftRecord {
5059            pre: pre.clone(),
5060            post: None,
5061            lines: None,
5062            limit: None,
5063            agent: None,
5064            outcome: GraftOutcome::Running,
5065            requested: graft::requested(&plan),
5066            adoption: None,
5067            adoption_error: None,
5068        });
5069        self.state.event(
5070            "graft",
5071            format!(
5072                "porting strengths of candidate(s) {} onto {}",
5073                plan.strengths
5074                    .iter()
5075                    .map(|(l, _, _)| l.to_string())
5076                    .collect::<Vec<_>>()
5077                    .join(", "),
5078                plan.winner
5079            ),
5080        );
5081        self.state.save()?;
5082
5083        let fix_prompt = prompt::graft(
5084            &self.state.instruction,
5085            plan.winner,
5086            &plan.strengths,
5087            &self.state.config.graph.language,
5088        );
5089        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5090        let sessions = self.state.config.graph.sessions;
5091        let artifacts = agent::artifacts_dir(&self.state.dir());
5092        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5093        let (_, seat, out) = self
5094            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5095                SeatJob {
5096                    prompt: fix_prompt.clone(),
5097                    spec,
5098                    seat,
5099                    cwd: wt.clone(),
5100                    timeout,
5101                    allow_write: true,
5102                    sessions,
5103                    artifacts: artifacts.clone(),
5104                    stem: "graft".to_owned(),
5105                    handover: None,
5106                }
5107            })
5108            .await;
5109        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5110            // Preserve the need to restore earlier fallback edits without
5111            // confusing an unstarted seat with an interrupted invocation.
5112            self.state.graft.as_mut().expect("graft record").outcome =
5113                GraftOutcome::WaitingForRestore;
5114            return self.stop_for_resource();
5115        }
5116        let mut why: Option<String> = None;
5117        match &out {
5118            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5119                why = Some("the fixer produced nothing usable".to_owned());
5120            }
5121            AgentOutcome::Ok(_) => {}
5122            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5123            AgentOutcome::Quota(o) => {
5124                self.state.quota.push(QuotaLoss::from_report(
5125                    seat.key.clone(),
5126                    "graft",
5127                    o.quota.as_ref(),
5128                ));
5129                why = Some("rate limited (quota)".to_owned());
5130            }
5131            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5132                why = Some(format!(
5133                    "the fixer failed: {}",
5134                    e.lines()
5135                        .next()
5136                        .unwrap_or("")
5137                        .chars()
5138                        .take(160)
5139                        .collect::<String>()
5140                ));
5141            }
5142        }
5143        let agent_id = seat.agent.clone();
5144        self.state.seats.insert(seat.key.clone(), seat);
5145
5146        let mut post = None;
5147        let mut lines = None;
5148        let _coordination = self.coordinate(&wt).await?;
5149        if why.is_none() {
5150            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5151            if let Ok(r) = &rescued {
5152                self.state.note_withheld("graft", &r.withheld);
5153            }
5154            let head = git::rev_parse(&wt, "HEAD").await?;
5155            // What is measured must be what gets reviewed and gated: a failed
5156            // rescue, or anything still uncommitted (withheld lockfiles
5157            // aside, which are never part of the tree), rejects the graft.
5158            let dirty = match &rescued {
5159                Ok(r) => git::status_porcelain(&wt)
5160                    .await
5161                    .map(|st| {
5162                        st.lines().any(|l| {
5163                            !r.withheld
5164                                .iter()
5165                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5166                        })
5167                    })
5168                    .unwrap_or(true),
5169                Err(_) => true,
5170            };
5171            if dirty {
5172                why = Some("the fixer left changes that could not be committed".to_owned());
5173            } else if head == pre {
5174                why = Some("the fixer changed nothing".to_owned());
5175            } else {
5176                lines = git::changed_lines(&wt, &pre, &head).await;
5177                post = Some(head);
5178            }
5179        }
5180        let (adoption, adoption_error) = match &out {
5181            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5182            _ => (None, None),
5183        };
5184        let requested = graft::requested(&plan);
5185        let outcome = match why {
5186            None => {
5187                self.state.event(
5188                    "graft",
5189                    format!(
5190                        "applied: {}",
5191                        lines.map_or_else(
5192                            || "changed lines unavailable".to_owned(),
5193                            |n| format!("{n} changed lines")
5194                        )
5195                    ),
5196                );
5197                GraftOutcome::Applied
5198            }
5199            Some(why) => {
5200                // Not settled until the restore is confirmed: the record
5201                // stays `Running`, so a resume retries it.
5202                crate::recovery::restore_repair(&wt, &pre)
5203                    .await
5204                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5205                self.state.event(
5206                    "graft",
5207                    format!("rolled back: {why}; the winner is untouched"),
5208                );
5209                GraftOutcome::RolledBack(why)
5210            }
5211        };
5212        self.state.graft = Some(GraftRecord {
5213            pre,
5214            post,
5215            lines,
5216            limit: None,
5217            agent: Some(agent_id),
5218            outcome,
5219            requested,
5220            adoption,
5221            adoption_error,
5222        });
5223        self.state.save()?;
5224        Ok(())
5225    }
5226
5227    // ----------------------------------------------------------------- fold
5228
5229    async fn fold_losers(&mut self) -> Result<()> {
5230        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5231            return Ok(());
5232        };
5233        if !self.state.candidate_preservation.is_empty() {
5234            bail!("candidate preservation incomplete; refusing cleanup");
5235        }
5236        let repo = self.state.repo.clone();
5237        let _coordination = self.coordinate(&repo).await?;
5238        let mut folded = Vec::new();
5239        for i in 0..self.state.candidates.len() {
5240            let c = &self.state.candidates[i];
5241            if c.label == winner || c.folded {
5242                continue;
5243            }
5244            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5245            remove_preserved_worktree(&repo, &wt).await?;
5246            if git::branch_exists(&repo, &branch).await?
5247                && !git::branch_delete(&repo, &branch).await?
5248            {
5249                bail!("candidate branch removal refused");
5250            }
5251            self.state.candidates[i].folded = true;
5252            self.state.save()?;
5253            folded.push(label.to_string());
5254        }
5255        // The judges are finished; their checkouts are pure cost from here.
5256        let root = self.state.worktree_root();
5257        for j in 1..=self.roles.judges.len() {
5258            let wt = root.join(format!("judge-{j}"));
5259            if wt.exists() {
5260                remove_preserved_worktree(&repo, &wt).await?;
5261            }
5262        }
5263        // The design-deliberation stage is finished by the time a tally
5264        // exists — same reasoning as the judges above.
5265        if self.state.config.graph.advise {
5266            for k in 1..=self.state.config.graph.advisors {
5267                let wt = root.join(format!("advisor-{k}"));
5268                if wt.exists() {
5269                    remove_preserved_worktree(&repo, &wt).await?;
5270                }
5271            }
5272        }
5273        if !folded.is_empty() {
5274            self.state
5275                .event("fold", format!("folded candidates {}", folded.join(", ")));
5276            self.state.save()?;
5277        }
5278        Ok(())
5279    }
5280
5281    // ------------------------------------------------------------ base sync
5282
5283    async fn hold_base_sync(&mut self) -> Result<bool> {
5284        self.hold_base_sync_as(None).await
5285    }
5286
5287    async fn hold_base_sync_kind(
5288        &mut self,
5289        kind: crate::base_sync_recovery::StopKind,
5290    ) -> Result<()> {
5291        self.hold_base_sync_as(Some(kind)).await?;
5292        Ok(())
5293    }
5294
5295    async fn hold_base_sync_as(
5296        &mut self,
5297        kind: Option<crate::base_sync_recovery::StopKind>,
5298    ) -> Result<bool> {
5299        if !self.state.base_sync_blocked() {
5300            self.state.base_sync_recovery.stop = None;
5301            return Ok(false);
5302        }
5303        let head = match self.state.winner() {
5304            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5305                .await
5306                .unwrap_or_default(),
5307            None => String::new(),
5308        };
5309        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5310            && !authorization.consumed
5311            && authorization.stop.head != head
5312        {
5313            authorization.refuse(format!(
5314                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5315                authorization.stop.head
5316            ));
5317        }
5318        self.state.record_base_sync_stop(&head);
5319        if let Some(kind) = kind {
5320            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5321            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5322                stop.kind = kind;
5323            }
5324        }
5325        self.state.save()?;
5326        Ok(true)
5327    }
5328
5329    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5330    /// anything verifies it.
5331    ///
5332    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5333    /// read whatever is checked out in the winner's worktree. Left alone that
5334    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5335    /// it when the run *branched* - and a run takes long enough that the base
5336    /// has usually moved by the time it gets here. A gate that ran there
5337    /// answers "green on the commit this run started from", not "green on
5338    /// what is about to land", and the difference showed up three times in
5339    /// one day as a green run whose merge would have reverted a file another
5340    /// pull request had already landed.
5341    ///
5342    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5343    /// `land::Step::Rebase` calls, rather than a second implementation of the
5344    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5345    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5346    /// doc warns about ("two answers to a question nobody notices until a
5347    /// diff is wrong").
5348    ///
5349    /// Foreign-checkout reviews instead merge the base into the owned shadow
5350    /// worktree, preserving the ancestry required for fast-forward delivery.
5351    /// A conflicting merge is aborted and held for operator resolution.
5352    ///
5353    /// A conflict is not the end of the road: the standing rebase is handed
5354    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5355    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5356    /// land). Once it finishes, review and the gate run as usual on the
5357    /// rebased tree, which is where a breakage the new base caused is caught
5358    /// by the ordinary gate-fix round. magi resolves nothing itself.
5359    ///
5360    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5361    /// the current durable recovery interval, bounds how often the base is
5362    /// synchronized during active recovery (lifetime attempts remain evidence).
5363    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5364    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5365    /// what was tried (rounds spent, paths still conflicted) and the branch
5366    /// and worktree stay exactly as they were - untouched, for a person to
5367    /// look at - which is also what makes re-entering this function
5368    /// afterwards a no-op instead of a second attempt at the same wall. A
5369    /// push failure ends the same way.
5370    async fn sync_to_base(&mut self) -> Result<()> {
5371        if self.state.status == RunStatus::AlreadyInBase {
5372            return Ok(());
5373        }
5374        self.hold_base_sync().await?;
5375        let conflicted = self
5376            .state
5377            .base_sync
5378            .as_ref()
5379            .is_some_and(|s| s.conflict.is_some());
5380        let Some(winner) = self.state.winner().cloned() else {
5381            return Ok(());
5382        };
5383
5384        let repo = self.state.repo.clone();
5385        let remote = self.state.config.merge.remote.clone();
5386        let base_branch = self.state.base_branch.clone();
5387        let tracking = format!("{remote}/{base_branch}");
5388
5389        let fetched_base =
5390            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5391        // No network, or the remote never had this branch: the run already
5392        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5393        // otherwise), and one that got this far is not blocked by a fetch
5394        // that fails now. It just has no newer tip to compare against.
5395        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5396            return Ok(());
5397        };
5398
5399        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5400        self.state.ensure_base_episode();
5401        let mut pending = self
5402            .state
5403            .base_sync_recovery
5404            .operations
5405            .last()
5406            .filter(|o| !o.finished)
5407            .cloned();
5408        let scratch = self.state.dir().join("base-sync");
5409        if git::rebase_in_progress(&scratch).await {
5410            // Older resource stops predate operation reservations. Import the
5411            // standing Git operation, never retarget it to today's main.
5412            if pending.is_none()
5413                && self.state.base_sync_recovery.operations.is_empty()
5414                && !conflicted
5415                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5416                    .await
5417                    .ok()
5418                    .as_deref()
5419                    == Some(head.as_str())
5420                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5421                && git::is_clean(&winner.worktree).await?
5422            {
5423                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5424                if self.state.base_sync.is_none() {
5425                    self.state.base_sync = Some(BaseSync {
5426                        tip: saved_tip.clone(),
5427                        behind: 0,
5428                        attempts: 0,
5429                        conflict: None,
5430                        already_in: None,
5431                    });
5432                }
5433                self.state.reserve_base_operation(&head, &saved_tip);
5434                self.state.event(
5435                    "base-recovery",
5436                    "imported standing legacy rebase as a spent synchronization reservation",
5437                );
5438                self.state.save()?;
5439                pending = self.state.base_sync_recovery.operations.last().cloned();
5440            }
5441            let expected = pending.as_ref().map(|op| op.tip.as_str());
5442            if expected.is_none()
5443                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5444            {
5445                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5446                if self.state.base_sync.is_none() {
5447                    self.state.base_sync = Some(BaseSync {
5448                        tip: tip.clone(),
5449                        behind: 0,
5450                        attempts: 0,
5451                        conflict: None,
5452                        already_in: None,
5453                    });
5454                }
5455                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5456                self.state.event("land", why);
5457                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5458                    .await?;
5459                return Ok(());
5460            }
5461        }
5462        if let Some(op) = &pending {
5463            // A replay may add the pinned base's commits, never foreign work.
5464            let mut replay_proven =
5465                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5466            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5467                let wait = repository_wait(&self.state, None);
5468                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5469                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5470                    Ok(true) => {
5471                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5472                        self.state.save()?;
5473                        git::worktree_remove(&repo, &scratch).await.ok();
5474                        replay_proven = true;
5475                    }
5476                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5477                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5478                }
5479            }
5480            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5481                if self.state.foreign_checkout.is_some() {
5482                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5483                        .await
5484                        .ok()
5485                        .as_deref()
5486                        == Some(op.head.as_str())
5487                        && git::rev_parse(&repo, &format!("{head}^2"))
5488                            .await
5489                            .ok()
5490                            .as_deref()
5491                            == Some(op.tip.as_str());
5492                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5493                    .await
5494                    .is_ok_and(|m| m.is_empty())
5495                    && let Ok(added) =
5496                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5497                {
5498                    replay_proven = true;
5499                    for commit in added {
5500                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5501                            replay_proven = false;
5502                            break;
5503                        }
5504                    }
5505                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5506                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5507                }
5508            }
5509            // Never retarget an operation across a crash, even if main moved.
5510            // sync_to_head resets and cleans with -x, so even ignored untracked
5511            // files must stop recovery independently of the old tracked tree.
5512            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5513                || (head != op.head
5514                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5515                        .await?
5516                        .is_empty())
5517                || (!git::is_clean(&winner.worktree).await?
5518                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5519                        .await
5520                        .is_ok_and(|o| o.ok()))
5521                || !replay_proven
5522            {
5523                let why = format!(
5524                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5525                    op.id
5526                );
5527                let sync = self
5528                    .state
5529                    .base_sync
5530                    .as_mut()
5531                    .context("reserved synchronization has no lifetime accounting")?;
5532                sync.conflict = Some(why.clone());
5533                self.state.event("land", why);
5534                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5535                    .await?;
5536                return Ok(());
5537            }
5538            if head != op.head {
5539                git::sync_to_head(&winner.worktree).await?;
5540            }
5541            tip = op.tip.clone();
5542        }
5543        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5544        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5545        if pending.is_none()
5546            && !conflicted
5547            && fetched_base
5548            && self
5549                .state
5550                .base_sync_recovery
5551                .episodes
5552                .last()
5553                .is_some_and(|e| e.boundary.is_some())
5554            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5555            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5556            && owner.runs.last() == Some(&self.state.id)
5557            && owner.status != crate::queue::TaskStatus::Done
5558            && !owner.fresh_start
5559            && owner.review_branch.is_none()
5560            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5561            && let Some(approval) = owner.resumption.as_ref()
5562            && approval.instruction == owner.instruction
5563            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5564            && git::is_clean(&winner.worktree).await?
5565            && self.state.renew_base_episode(&task, approval, &head, &tip)
5566        {
5567            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5568            self.state.save()?;
5569        }
5570
5571        // Before any rebase, and before a recorded conflict is honoured: a
5572        // branch whose change reached the base under other commit ids has
5573        // nothing to rebase and nothing to conflict with, and a run that
5574        // already stopped on that phantom conflict recovers here on resume.
5575        // `behind == 0` with head == tip is a branch the base has since taken
5576        // in whole, whether or not a conflict was ever recorded: the ancestry
5577        // proof must still run (`classify` ignores a head still on the start
5578        // commit).
5579        if (behind > 0 || conflicted || head == tip)
5580            && self
5581                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5582                .await?
5583        {
5584            return Ok(());
5585        }
5586        if conflicted && self.state.base_recovery_pending() {
5587            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5588            let owner =
5589                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5590            let owner_valid = owner.as_ref().is_ok_and(|t| {
5591                t.runs.last() == Some(&self.state.id)
5592                    && t.status != crate::queue::TaskStatus::Done
5593                    && !t.fresh_start
5594                    && t.review_branch.is_none()
5595            });
5596            let invalidation = if !owner_valid {
5597                Some("latest task ownership changed")
5598            } else if !owner.as_ref().is_ok_and(|t| {
5599                authorization
5600                    .generation
5601                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5602            }) {
5603                Some("owner approval generation changed")
5604            } else if let Some(scope) = &authorization.scope {
5605                if scope.tip != tip {
5606                    Some("fetched base changed")
5607                } else if !self
5608                    .state
5609                    .base_sync_recovery
5610                    .episodes
5611                    .last()
5612                    .is_some_and(|e| e.id == scope.episode)
5613                {
5614                    Some("active synchronization interval changed")
5615                } else if !owner
5616                    .as_ref()
5617                    .is_ok_and(|t| t.instruction == scope.instruction)
5618                {
5619                    Some("task instruction changed")
5620                } else {
5621                    None
5622                }
5623            } else {
5624                None
5625            };
5626            if let Some(reason) = invalidation {
5627                let authorization = self
5628                    .state
5629                    .base_sync_recovery
5630                    .authorizations
5631                    .last_mut()
5632                    .unwrap();
5633                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5634                self.state.save()?;
5635                return Ok(());
5636            }
5637            if !fetched_base
5638                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5639                || !git::is_clean(&winner.worktree).await?
5640                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5641            {
5642                return Ok(());
5643            }
5644        }
5645        let recovery = conflicted && self.state.consume_base_recovery(&head);
5646        if conflicted {
5647            // Consumption (including a refused changed HEAD) is durable before Git.
5648            self.state.save()?;
5649            if !recovery {
5650                return Ok(());
5651            }
5652        }
5653
5654        if behind == 0 {
5655            // A fixer-finished rebase moves the branch ref before the
5656            // winner's worktree is told (`sync_to_head` below). A run that
5657            // died in between resumes here with `behind == 0` and a tree still
5658            // holding the pre-rebase files, which review and the gate would
5659            // then read. That state is exactly: HEAD moved off the tip the
5660            // rebase started from, yet the tree is still identical to that
5661            // tip. A tree with edits of its own differs from it, so nothing
5662            // is thrown away.
5663            if let Some(from) = self
5664                .state
5665                .rebase_fixes
5666                .iter()
5667                .rev()
5668                .find_map(|r| r.from.clone())
5669                && from != head
5670                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5671                    .await
5672                    .is_ok_and(|o| o.ok())
5673            {
5674                git::sync_to_head(&winner.worktree).await?;
5675            }
5676            // An earlier attempt may have rebased the branch locally and died
5677            // before pushing it (only the fresh-rebase arm below pushes).
5678            // Publish it now, so the plain push at PR time is not refused as
5679            // a non-fast-forward. A run already holding a recorded conflict
5680            // never reaches here; that case is out of scope.
5681            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5682            if let Some(why) = &conflict {
5683                self.state.status = RunStatus::Blocked;
5684                self.state.event("land", why.clone());
5685            }
5686            self.state.base_sync = Some(BaseSync {
5687                tip,
5688                behind: 0,
5689                attempts,
5690                conflict,
5691                already_in: None,
5692            });
5693            if pending.is_some() {
5694                self.state.finish_base_operation(Some(&head));
5695            }
5696            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5697                .await?;
5698            self.state.save()?;
5699            return Ok(());
5700        }
5701
5702        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5703            let why = format!(
5704                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5705                 synchronization(s); synchronizing again would only race it",
5706                winner.branch
5707            );
5708            self.state.status = RunStatus::Blocked;
5709            self.state.base_sync = Some(BaseSync {
5710                tip,
5711                behind,
5712                attempts,
5713                conflict: Some(why.clone()),
5714                already_in: None,
5715            });
5716            self.state.event("land", why);
5717            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5718                .await?;
5719            self.state.save()?;
5720            return Ok(());
5721        }
5722
5723        if self.state.foreign_checkout.is_some() {
5724            // Delivery to the operator's checkout is fast-forward only. A
5725            // rebase here would discard its saved head from our ancestry.
5726            let wait = repository_wait(&self.state, None);
5727            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5728            if self.state.base_sync.is_none() {
5729                self.state.base_sync = Some(BaseSync {
5730                    tip: tip.clone(),
5731                    behind,
5732                    attempts: 0,
5733                    conflict: None,
5734                    already_in: None,
5735                });
5736            }
5737            self.state.reserve_base_operation(&head, &tip);
5738            self.state.save()?;
5739            let merged = git::git_raw(
5740                &winner.worktree,
5741                &[
5742                    "-c",
5743                    "core.hooksPath=/dev/null",
5744                    "merge",
5745                    "--no-ff",
5746                    "--no-autostash",
5747                    "--no-edit",
5748                    "-m",
5749                    &format!("Merge {tracking} before verifying"),
5750                    &tip,
5751                ],
5752            )
5753            .await?;
5754            let conflict = if merged.ok() {
5755                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5756                self.state.event(
5757                    "land",
5758                    format!("merged {tracking} into {} before verifying", winner.branch),
5759                );
5760                None
5761            } else {
5762                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5763                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5764                }
5765                let why = format!(
5766                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5767                    winner.branch,
5768                    merged.stderr.chars().take(600).collect::<String>()
5769                );
5770                self.state.status = RunStatus::Blocked;
5771                self.state.event("land", &why);
5772                Some(why)
5773            };
5774            let synced = conflict.is_none();
5775            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5776            self.state
5777                .finish_base_operation(synced.then_some(fresh.as_str()));
5778            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5779            self.state.base_sync = Some(BaseSync {
5780                tip,
5781                behind: if synced { 0 } else { behind },
5782                attempts,
5783                conflict,
5784                already_in: None,
5785            });
5786            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5787                .await?;
5788            self.state.save()?;
5789            return Ok(());
5790        }
5791
5792        self.state.event(
5793            "land",
5794            format!(
5795                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5796                winner.branch
5797            ),
5798        );
5799        self.state.save()?;
5800
5801        // The remote's copy of the branch, read now and only if the fetch
5802        // really succeeded (a stale tracking ref must never pin a lease). It is
5803        // pushed over after a rebase only when it is a commit this branch
5804        // already contains, by ancestry or by patch (an earlier rebase of ours
5805        // that never reached the remote): anything else is somebody else's work.
5806        let branch_tracking = format!("{remote}/{}", winner.branch);
5807        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5808        let remote_tip = if self.state.task_pr_branch().is_none()
5809            && matches!(&fetched_branch, Ok(o) if o.ok())
5810        {
5811            git::rev_parse(&repo, &branch_tracking).await.ok()
5812        } else {
5813            None
5814        };
5815        // A remote tip this branch does not contain is somebody else's work:
5816        // rebasing would leave a local tip that can never be pushed. Stop
5817        // before touching anything and say so.
5818        if let Some(theirs) = &remote_tip
5819            && !git::is_ancestor(&repo, theirs, &head).await
5820            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5821                .await
5822                .is_ok_and(|missing| missing.is_empty())
5823        {
5824            let why = format!(
5825                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5826                 them",
5827                short(theirs),
5828                winner.branch
5829            );
5830            self.state.status = RunStatus::Blocked;
5831            self.state.base_sync = Some(BaseSync {
5832                tip,
5833                behind,
5834                attempts,
5835                conflict: Some(why.clone()),
5836                already_in: None,
5837            });
5838            self.state.event("land", why);
5839            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5840                .await?;
5841            self.state.save()?;
5842            return Ok(());
5843        }
5844
5845        if self.state.base_sync.is_none() {
5846            self.state.base_sync = Some(BaseSync {
5847                tip: tip.clone(),
5848                behind,
5849                attempts: 0,
5850                conflict: None,
5851                already_in: None,
5852            });
5853        }
5854        self.state.reserve_base_operation(&head, &tip);
5855        self.state.save()?;
5856        let scratch = self.state.dir().join("base-sync");
5857        let rebased =
5858            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5859                .await
5860            {
5861                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5862                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5863                Err(e) => Err(e),
5864            };
5865        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5866        let stop_kind = match &rebased {
5867            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5868            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5869            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5870        };
5871        match rebased {
5872            Ok(None) => {
5873                // The branch ref moved, but a worktree that already had it
5874                // checked out (the winner's) was not told; sync its index and
5875                // files before anything reads them.
5876                git::sync_to_head(&winner.worktree).await?;
5877                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5878                self.state.finish_base_operation(Some(&fresh));
5879                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5880                let mut conflict = None;
5881                if let Some(pinned) = &remote_tip {
5882                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5883                    match pushed {
5884                        Ok(o) if o.ok() => self.state.event(
5885                            "land",
5886                            format!("pushed rebased {} to {remote}", winner.branch),
5887                        ),
5888                        Ok(o) => {
5889                            conflict = Some(format!(
5890                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5891                                winner.branch,
5892                                short(pinned),
5893                                o.stderr.chars().take(600).collect::<String>()
5894                            ));
5895                        }
5896                        Err(e) => {
5897                            conflict = Some(format!(
5898                                "rebased {} locally but could not push it: {e:#}",
5899                                winner.branch
5900                            ));
5901                        }
5902                    }
5903                }
5904                if let Some(why) = &conflict {
5905                    self.state.status = RunStatus::Blocked;
5906                    self.state.event("land", why.clone());
5907                }
5908                self.state.base_sync = Some(BaseSync {
5909                    tip: tip.clone(),
5910                    behind: 0,
5911                    attempts,
5912                    conflict,
5913                    already_in: None,
5914                });
5915                self.state
5916                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5917            }
5918            Ok(Some(conflict)) => {
5919                self.state.finish_base_operation(None);
5920                let why = format!(
5921                    "{} conflicts with {tracking} and did not rebase: {}",
5922                    winner.branch,
5923                    conflict.chars().take(600).collect::<String>()
5924                );
5925                self.state.status = RunStatus::Blocked;
5926                self.state.base_sync = Some(BaseSync {
5927                    tip,
5928                    behind,
5929                    attempts,
5930                    conflict: Some(why.clone()),
5931                    already_in: None,
5932                });
5933                self.state.event("land", why);
5934            }
5935            // A refused Git write or a busy repository: nothing is wrong with
5936            // the branch. The scratch rebase stays standing and the run
5937            // resumes into it (`base_sync` must not claim a conflict).
5938            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5939                self.state.save()?;
5940                return Ok(());
5941            }
5942            Err(e) => {
5943                self.state.finish_base_operation(None);
5944                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5945                self.state.status = RunStatus::Blocked;
5946                self.state.base_sync = Some(BaseSync {
5947                    tip,
5948                    behind,
5949                    attempts,
5950                    conflict: Some(why.clone()),
5951                    already_in: None,
5952                });
5953                self.state.event("land", why);
5954            }
5955        }
5956        if !self.state.base_sync_blocked() {
5957            self.state.base_sync_recovery.stop = None;
5958        }
5959        self.hold_base_sync_kind(stop_kind).await?;
5960        self.state.save()?;
5961        Ok(())
5962    }
5963
5964    /// Push a branch an earlier attempt rebased locally but never published,
5965    /// pinned to the remote tip read right after a successful fetch. Returns
5966    /// the reason when the run must stop; `None` when there was nothing to do
5967    /// (no remote copy, the same tip, or a remote copy this branch already
5968    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5969    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5970        // Task competitions publish only after the PR text gate, to their
5971        // public head; never resurrect a candidate branch on the remote.
5972        if self.state.task_pr_branch().is_some() {
5973            return None;
5974        }
5975        let repo = self.state.repo.clone();
5976        let remote = self.state.config.merge.remote.clone();
5977        let fetched = git::fetch(&repo, &remote, branch).await;
5978        if !matches!(&fetched, Ok(o) if o.ok()) {
5979            return None;
5980        }
5981        let branch_tracking = format!("{remote}/{branch}");
5982        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5983        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5984            return None;
5985        }
5986        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5987            .await
5988            .is_ok_and(|missing| missing.is_empty())
5989        {
5990            return Some(format!(
5991                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5992                 them",
5993                short(&theirs)
5994            ));
5995        }
5996        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5997            Ok(o) if o.ok() => {
5998                self.state
5999                    .event("land", format!("pushed rebased {branch} to {remote}"));
6000                None
6001            }
6002            Ok(o) => Some(format!(
6003                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6004                 someone may have pushed): {}",
6005                short(&theirs),
6006                o.stderr.chars().take(600).collect::<String>()
6007            )),
6008            Err(e) => Some(format!(
6009                "{branch} is rebased locally but could not be pushed: {e:#}"
6010            )),
6011        }
6012    }
6013
6014    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6015    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6016    /// whether it did.
6017    ///
6018    /// Checked only when the base is ahead of the branch. A failing check is
6019    /// "not proven" - the ordinary rebase path then decides - never a reason to
6020    /// stop the run.
6021    ///
6022    /// The remote copy of the branch is held to the same standard as the local
6023    /// one: if it carries a tip this worktree does not, that tip must itself be
6024    /// proven in the base, or nothing is settled (a pull request would
6025    /// otherwise be closed over commits nobody checked). The pull request is
6026    /// closed *before* the terminal status is saved; if that fails for a
6027    /// reason other than a refusal (no network, a `gh` error) the run is left
6028    /// `Blocked` with the reason as its conflict, which a resume retries -
6029    /// the same recovery a phantom conflict gets.
6030    async fn settle_already_in(
6031        &mut self,
6032        branch: &str,
6033        tip: &str,
6034        head: &str,
6035        attempts: usize,
6036        behind: usize,
6037    ) -> Result<bool> {
6038        let repo = self.state.repo.clone();
6039        let remote = self.state.config.merge.remote.clone();
6040        let start = self.state.base_commit.clone();
6041        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6042            Ok(Some(e)) => e,
6043            Ok(None) => return Ok(false),
6044            Err(e) => {
6045                tracing::warn!("already-in-base check for {branch}: {e:#}");
6046                return Ok(false);
6047            }
6048        };
6049        let mut verified = vec![head.to_owned()];
6050        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6051        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6052        if matches!(&fetched, Ok(o) if o.ok())
6053            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6054            && theirs != head
6055        {
6056            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6057                Ok(Some(_)) => verified.push(theirs),
6058                _ => return Ok(false),
6059            }
6060        }
6061        let base_branch = self.state.base_branch.clone();
6062        let message = format!(
6063            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6064             land",
6065            evidence.names(),
6066            evidence.proof.as_str()
6067        );
6068        let closed =
6069            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6070                .await;
6071        match closed {
6072            Ok(Ok(url)) => self
6073                .state
6074                .event("land", format!("closed {url}: superseded on {base_branch}")),
6075            Ok(Err(why)) => self
6076                .state
6077                .event("land", format!("did not close a pull request: {why}")),
6078            Err(e) => {
6079                let why = format!(
6080                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6081                     not be closed ({e:#}); resume to retry"
6082                );
6083                self.state.status = RunStatus::Blocked;
6084                self.state.base_sync = Some(BaseSync {
6085                    tip: tip.to_owned(),
6086                    behind,
6087                    attempts,
6088                    conflict: Some(why.clone()),
6089                    already_in: None,
6090                });
6091                self.state.event("land", why);
6092                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6093                    .await?;
6094                self.state.save()?;
6095                return Ok(true);
6096            }
6097        }
6098        self.state.status = RunStatus::AlreadyInBase;
6099        self.state.base_sync_recovery.stop = None;
6100        self.state.base_sync = Some(BaseSync {
6101            tip: tip.to_owned(),
6102            behind,
6103            attempts,
6104            conflict: None,
6105            already_in: Some(evidence),
6106        });
6107        self.state.event("land", message);
6108        self.state.save()?;
6109        self.settle_questions();
6110        Ok(true)
6111    }
6112
6113    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6114    /// last landed the winner on, once it has run, else the commit the run
6115    /// branched from.
6116    ///
6117    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6118    /// and `vote` all happen before there is a winner to rebase, so they
6119    /// compare every candidate against the branch point on purpose, and a
6120    /// base that moves after they are already done cannot change an answer
6121    /// they already gave.
6122    fn landing_base(&self) -> String {
6123        self.state
6124            .base_sync
6125            .as_ref()
6126            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6127    }
6128
6129    // ------------------------------------------------------- operator fix
6130
6131    /// Route specific, already-recorded review findings to a fixer for a
6132    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6133    /// entry point.
6134    ///
6135    /// Distinct from `review_loop`'s own fix step in three ways: it never
6136    /// runs a reviewer wave, it never spends review-round budget, and what
6137    /// happened is recorded as an [`OperatorFixRequest`] appended to
6138    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6139    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6140    /// and vote must never be rewritten to look like a manufactured blocking
6141    /// verdict.
6142    ///
6143    /// Only meaningful once review has actually concluded: `Ready` (handed
6144    /// off with findings still open, or simply concluded clean while minor
6145    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6146    /// gate failed). Everything else is refused: a run still in progress
6147    /// should simply be resumed, and a `Merged` run's branch has already
6148    /// landed — reopening *this* run's own record cannot change that, so the
6149    /// answer there is a fresh `magi review <branch>`.
6150    ///
6151    /// A real commit here re-verifies through a fresh, ordinary review-only
6152    /// run on the same branch ([`Self::review`]) rather than reopening this
6153    /// run's own `review_loop`: once any round in this run's history went
6154    /// clean, `review_conclusion` treats that as permanent by design (the
6155    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6156    /// way to force one more genuine reviewer wave out of *this* run without
6157    /// either rewriting history or weakening that guarantee for every other
6158    /// caller. A review-only run costs nothing extra — no implementation, no
6159    /// judging, no vote — and exercises the exact same review → verify →
6160    /// gate → (human) merge path, unmodified.
6161    pub async fn fix_selected(
6162        &mut self,
6163        ids: &[String],
6164        reason: &str,
6165        allow_stale: bool,
6166    ) -> Result<()> {
6167        let reason = reason.trim();
6168        if reason.is_empty() {
6169            bail!("a fix request needs a reason — that is the operator's own record of why");
6170        }
6171        if ids.is_empty() {
6172            bail!("no finding id given");
6173        }
6174        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6175            bail!(
6176                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6177                 has already concluded — can be given a targeted fix. A run still \
6178                 in progress should simply be resumed; a `merged` run's branch has \
6179                 already landed, so its answer is a fresh `magi review <branch>`, \
6180                 not reopening this run's own record",
6181                self.state.id,
6182                self.state.status.as_str()
6183            );
6184        }
6185        let Some(winner) = self.state.winner().cloned() else {
6186            bail!("run {} has no winning candidate to fix", self.state.id);
6187        };
6188        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6189            bail!(
6190                "branch `{}` no longer exists; this run cannot be extended",
6191                winner.branch
6192            );
6193        }
6194        let home = crate::run::home();
6195        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6196            bail!(
6197                "run {} is currently being worked on by another magi process",
6198                self.state.id
6199            );
6200        }
6201        // Held for the rest of this call, including the follow-up review
6202        // below: two `magi fix` invocations against the same run must not
6203        // both reach the worktree manipulation further down, which would
6204        // otherwise race to remove and recreate the same directory — see
6205        // [`FixClaim`]'s own doc.
6206        let _claim = FixClaim::acquire(&self.state.dir())?;
6207
6208        // Resolve every id before spending anything — an unknown id refuses
6209        // the whole request rather than silently dropping it — and dedup
6210        // while keeping the operator's own order.
6211        let mut seen = BTreeSet::new();
6212        let mut findings = Vec::new();
6213        let mut missing = Vec::new();
6214        for id in ids {
6215            if !seen.insert(id.clone()) {
6216                continue;
6217            }
6218            match self.state.finding(id) {
6219                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6220                    id: f.id.clone(),
6221                    severity: f.severity,
6222                    reviewer_vote: rec.vote,
6223                    round: round.round,
6224                    round_head: round.head.clone(),
6225                    reviewer: rec.reviewer,
6226                    agent: rec.agent.clone(),
6227                    file: f.file.clone(),
6228                    line: f.line,
6229                    title: f.title.clone(),
6230                    detail: f.detail.clone(),
6231                    outcome: OperatorFixOutcome::Pending,
6232                }),
6233                None => missing.push(id.clone()),
6234            }
6235        }
6236        if !missing.is_empty() {
6237            bail!(
6238                "unknown finding id(s): {}; nothing was changed",
6239                missing.join(", ")
6240            );
6241        }
6242
6243        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6244        let stale_details: Vec<(String, String)> = findings
6245            .iter()
6246            .filter(|f| f.round_head != head_at_request)
6247            .map(|f| (f.id.clone(), f.round_head.clone()))
6248            .collect();
6249        let stale = !stale_details.is_empty();
6250        if stale && !allow_stale {
6251            bail!(
6252                "the branch has moved since some finding(s) were raised — {} — now \
6253                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6254                stale_details
6255                    .iter()
6256                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6257                    .collect::<Vec<_>>()
6258                    .join(", "),
6259                short(&head_at_request)
6260            );
6261        }
6262
6263        let request = OperatorFixRequest {
6264            requested_at: Timestamp::now(),
6265            reason: reason.to_owned(),
6266            findings,
6267            head_at_request: head_at_request.clone(),
6268            allow_stale,
6269            stale,
6270            fix: None,
6271            result_head: None,
6272            follow_up_review_run: None,
6273        };
6274        self.state.event(
6275            "fix",
6276            format!(
6277                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6278                request.findings.len(),
6279                request
6280                    .findings
6281                    .iter()
6282                    .map(|f| f.id.as_str())
6283                    .collect::<Vec<_>>()
6284                    .join(", "),
6285            ),
6286        );
6287        // Recorded now, before any worktree work or the fixer call itself —
6288        // and re-saved at each checkpoint below: a crash at any point after
6289        // this (mid fixer call, mid follow-up review) must not lose the fact
6290        // that this was requested, for which findings, and why. Everything
6291        // past this point reads and writes through `request_index` rather
6292        // than a local variable, since `request` itself is moved here.
6293        self.state.operator_fixes.push(request);
6294        self.state.save()?;
6295        let request_index = self.state.operator_fixes.len() - 1;
6296
6297        // A fresh, dedicated worktree for this one call, never the winner's
6298        // own worktree in place: that one may already be gone (folded away),
6299        // and reusing it in place would leave the branch checked out there
6300        // when the follow-up review below tries to check it out again. Freed
6301        // immediately after, either way — but only once confirmed clean:
6302        // `worktree_remove` is a `git worktree remove --force`, which would
6303        // otherwise discard uncommitted work left there by the operator or
6304        // another process before this had a chance to even look at it.
6305        let repo = self.state.repo.clone();
6306        let coordination = self.coordinate(&repo).await?;
6307        if winner.worktree.exists() {
6308            // Lockfiles a rescue commit withheld stay untracked on purpose and
6309            // are already recorded; they are not the operator's work to protect.
6310            let dirty = git::git(
6311                &winner.worktree,
6312                &["status", "--porcelain", "--untracked-files=all"],
6313            )
6314            .await?;
6315            let only_withheld = dirty.lines().all(|l| {
6316                l.strip_prefix("?? ")
6317                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6318            });
6319            if !only_withheld {
6320                bail!(
6321                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6322                     discard them first",
6323                    winner.worktree.display()
6324                );
6325            }
6326            git::worktree_remove(&self.state.repo, &winner.worktree)
6327                .await
6328                .ok();
6329        }
6330        let fix_worktree = self.state.worktree_root().join("operator-fix");
6331        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6332        git::git(
6333            &self.state.repo,
6334            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6335        )
6336        .await
6337        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6338        if !git::is_clean(&fix_worktree).await? {
6339            git::worktree_remove(&self.state.repo, &fix_worktree)
6340                .await
6341                .ok();
6342            bail!(
6343                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6344                winner.branch
6345            );
6346        }
6347        let repo_for_setup = self.state.repo.clone();
6348        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6349            Ok(()) => {
6350                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6351            }
6352            Err(e) => Err(e),
6353        };
6354        if let Err(e) = setup {
6355            git::worktree_remove(&self.state.repo, &fix_worktree)
6356                .await
6357                .ok();
6358            return Err(e);
6359        }
6360
6361        // This explicit operator request retries availability independently
6362        // of execute(); the previous call has already saved and cleaned up.
6363        self.state.resource_block = None;
6364
6365        let run_id = self.state.id.clone();
6366        let prompts = self.state.config.prompts.clone();
6367        let language = self.state.config.graph.language.clone();
6368        let sessions = self.state.config.graph.sessions;
6369        let artifacts = agent::artifacts_dir(&self.state.dir());
6370        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6371            .findings
6372            .iter()
6373            .map(|f| Finding {
6374                id: f.id.clone(),
6375                severity: f.severity,
6376                file: f.file.clone(),
6377                line: f.line,
6378                title: f.title.clone(),
6379                detail: f.detail.clone(),
6380            })
6381            .collect();
6382        let fix_prompt = prompt::operator_fix_in(
6383            &self.state.instruction,
6384            &finding_list,
6385            reason,
6386            &stale_details,
6387            &head_at_request,
6388            &language,
6389            &self.state.config.graph.github_language,
6390        );
6391        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6392        drop(coordination);
6393        let (job, seat, out) = self
6394            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6395                prompt: fix_prompt.clone(),
6396                spec,
6397                seat,
6398                cwd: fix_worktree.clone(),
6399                timeout,
6400                allow_write: true,
6401                sessions,
6402                artifacts: artifacts.clone(),
6403                stem: "operator-fix".to_owned(),
6404                handover: None,
6405            })
6406            .await;
6407        let agent_id = seat.agent.clone();
6408
6409        let mut fix = FixRecord {
6410            agent: agent_id,
6411            addressed: Vec::new(),
6412            rejected: Vec::new(),
6413            notes: String::new(),
6414            committed: false,
6415            failed: None,
6416            duration_ms: 0,
6417            continuation: None,
6418        };
6419        let mut final_seat = seat.clone();
6420        match out {
6421            AgentOutcome::Ok(o) => {
6422                fix.duration_ms = o.duration_ms;
6423                let parsed = verdict::extract_json::<FixReport>(&o.text);
6424                let incomplete_reason = match &parsed {
6425                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6426                        "the reply parsed, but it reported a command whose own CLI \
6427                         never confirmed an exit status"
6428                            .to_owned(),
6429                    ),
6430                    Ok(_) => None,
6431                    Err(e) => Some(e.to_string()),
6432                };
6433                match incomplete_reason {
6434                    None => {
6435                        let report = parsed.expect("checked Ok above");
6436                        fix.addressed = report.addressed;
6437                        fix.rejected = report.rejected;
6438                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6439                    }
6440                    Some(reason) => {
6441                        let (resumed_seat, resolved, failure, cont) = self
6442                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6443                            .await;
6444                        fix.duration_ms += cont.cumulative_wait_ms;
6445                        fix.continuation = Some(cont);
6446                        final_seat = resumed_seat;
6447                        match resolved {
6448                            Some(report) => {
6449                                fix.addressed = report.addressed;
6450                                fix.rejected = report.rejected;
6451                                fix.notes =
6452                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6453                            }
6454                            None => fix.failed = failure,
6455                        }
6456                    }
6457                }
6458            }
6459            AgentOutcome::Dropped(o) => {
6460                fix.duration_ms = o.duration_ms;
6461                let why = o
6462                    .dropped
6463                    .as_ref()
6464                    .map(|d| d.why.as_str())
6465                    .unwrap_or("the CLI ended the stream without delivering its answer");
6466                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6467            }
6468            AgentOutcome::Quota(o) => {
6469                self.state.quota.push(QuotaLoss::from_report(
6470                    final_seat.key.clone(),
6471                    "fix",
6472                    o.quota.as_ref(),
6473                ));
6474                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6475            }
6476            AgentOutcome::Failed(e) => fix.failed = Some(e),
6477            AgentOutcome::ResourceBlocked(e) => {
6478                fix.failed = Some(format!("the fixer was not started: {e}"));
6479            }
6480        }
6481        // Even a deferred call must rescue edits, release the worktree and
6482        // finalize its record below so an explicit `magi fix` can retry.
6483        if fix.continuation.is_none() {
6484            fix.continuation = Some(ContinuationRecord::not_needed());
6485        }
6486        self.state.seats.insert(final_seat.key.clone(), final_seat);
6487
6488        let rescue_message = format!(
6489            "magi: operator-selected fix ({}) (uncommitted work)",
6490            self.state.operator_fixes[request_index]
6491                .findings
6492                .iter()
6493                .map(|f| f.id.as_str())
6494                .collect::<Vec<_>>()
6495                .join(", ")
6496        );
6497        let repo = self.state.repo.clone();
6498        let coordination = self.coordinate(&repo).await?;
6499        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6500            self.state.note_withheld("fix", &r.withheld);
6501        }
6502        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6503        fix.committed = after != head_at_request;
6504        git::worktree_remove(&self.state.repo, &fix_worktree)
6505            .await
6506            .ok();
6507        drop(coordination);
6508
6509        self.state.event(
6510            "fix",
6511            match &fix.failed {
6512                Some(reason) => format!(
6513                    "operator fix: adoption report was lost ({reason}); {}",
6514                    if fix.committed {
6515                        "committed"
6516                    } else {
6517                        "NO new commit"
6518                    }
6519                ),
6520                None => format!(
6521                    "operator fix: {} addressed, {} rejected, {}",
6522                    fix.addressed.len(),
6523                    fix.rejected.len(),
6524                    if fix.committed {
6525                        "committed"
6526                    } else {
6527                        "NO new commit"
6528                    }
6529                ),
6530            },
6531        );
6532
6533        // Every selected finding gets an outcome — never left `Pending` once
6534        // the fixer's own turn is over. A report that never came back at all
6535        // marks every one of them `Unreported`, not silently "not addressed":
6536        // quota, a dropped stream, or an exhausted continuation are gaps in
6537        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6538        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6539        for f in &mut self.state.operator_fixes[request_index].findings {
6540            f.outcome = if fix.failed.is_some() {
6541                OperatorFixOutcome::Unreported
6542            } else if fix.addressed.contains(&f.id) {
6543                OperatorFixOutcome::Addressed
6544            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6545                OperatorFixOutcome::Rejected { why: r.why.clone() }
6546            } else {
6547                OperatorFixOutcome::Unreported
6548            };
6549        }
6550
6551        let committed = fix.committed;
6552        if committed {
6553            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6554        }
6555        self.state.operator_fixes[request_index].fix = Some(fix);
6556        // Saved again now that the fixer's own outcome is final, on top of
6557        // the save right after the request was first pushed above.
6558        self.state.save()?;
6559        if self.state.resource_block.is_some() {
6560            return self.stop_for_resource();
6561        }
6562
6563        if committed {
6564            self.state.event(
6565                "fix",
6566                format!(
6567                    "operator fix committed {}; opening a follow-up review-only run",
6568                    short(&after)
6569                ),
6570            );
6571            // The operator asked for the fix, and the follow-up serves whatever
6572            // task the run it follows served.
6573            let origin =
6574                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6575            match Self::review(
6576                &self.state.repo,
6577                &winner.branch,
6578                self.state.config.clone(),
6579                origin,
6580            )
6581            .await
6582            {
6583                Ok(mut follow_up) => {
6584                    follow_up.state.event(
6585                        "start",
6586                        format!(
6587                            "requested by an operator fix on run {} for finding(s) {}",
6588                            self.state.id,
6589                            self.state.operator_fixes[request_index]
6590                                .findings
6591                                .iter()
6592                                .map(|f| f.id.as_str())
6593                                .collect::<Vec<_>>()
6594                                .join(", "),
6595                        ),
6596                    );
6597                    follow_up.state.save()?;
6598                    let follow_up_id = follow_up.state.id.clone();
6599                    // The follow-up is a run like any other: it belongs to the
6600                    // task of the run it follows, or to one filed for it.
6601                    let adopted = match crate::direct::adopt(
6602                        &follow_up.state,
6603                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6604                    ) {
6605                        Ok(a) => a,
6606                        Err(e) => {
6607                            // An ownerless run must not spend agent calls; it
6608                            // stays saved, and `magi run --resume` adopts it.
6609                            self.state.event(
6610                                "fix",
6611                                format!(
6612                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6613                                ),
6614                            );
6615                            self.state.save()?;
6616                            return Ok(());
6617                        }
6618                    };
6619                    let executed = follow_up.execute().await;
6620                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6621                    if let Some(a) = adopted {
6622                        a.finish(&follow_up.state, executed);
6623                    }
6624                    if let Some(e) = failure {
6625                        self.state.event(
6626                            "fix",
6627                            format!(
6628                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6629                            ),
6630                        );
6631                    }
6632                    self.state.operator_fixes[request_index].follow_up_review_run =
6633                        Some(follow_up_id);
6634                }
6635                Err(e) => {
6636                    self.state.event(
6637                        "fix",
6638                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6639                    );
6640                }
6641            }
6642            self.state.save()?;
6643        }
6644
6645        Ok(())
6646    }
6647
6648    // --------------------------------------------------------------- review
6649
6650    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6651    /// names, in order, each at most once, moving on only when the call
6652    /// advances (an error, a quota hit or nothing usable - the same decision
6653    /// point `agent::chain_advances` is for the other chained roles).
6654    ///
6655    /// Each agent gets its own seat from `Runner::seat`: the same agent
6656    /// continues its conversation (the winner's own implementer seat when it
6657    /// is the winner's author, now that the competition is over), another
6658    /// takes a fresh one so the full prompt is sent again. A handover is
6659    /// recorded under `node`, which is what makes the fallback stick for the
6660    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6661    /// is returned, so an exhausted chain reads exactly like a single failed
6662    /// fixer: one quota loss, the same wording, the same refund. An earlier
6663    /// attempt's edits are left in the tree and judged, with the final
6664    /// attempt's, by what git says afterwards.
6665    ///
6666    /// Returns the job that produced the outcome, for `continue_fix_report`.
6667    async fn ask_fixer(
6668        &mut self,
6669        winner: &Candidate,
6670        node: &'static str,
6671        round: Option<usize>,
6672        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6673    ) -> (SeatJob, SeatState, AgentOutcome) {
6674        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6675        self.ask_fixer_among(attempts, node, round, None, build)
6676            .await
6677    }
6678
6679    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6680    /// worktree and a commit: before the next agent is asked after one that
6681    /// did not satisfy the call, the worktree is put back on that commit, so
6682    /// a failed agent's partial edits are not carried into the next one.
6683    async fn ask_fixer_among(
6684        &mut self,
6685        attempts: Vec<(AgentSpec, String)>,
6686        node: &'static str,
6687        round: Option<usize>,
6688        restore: Option<(&Path, &str)>,
6689        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6690    ) -> (SeatJob, SeatState, AgentOutcome) {
6691        let run_id = self.state.id.clone();
6692        let prompts = self.state.config.prompts.clone();
6693        let cache = self.state.config.cache_dir();
6694        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6695        let mut last = None;
6696        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6697            let seat = self.seat(&key, &spec.id);
6698            let mut job = build(spec.clone(), seat);
6699            if i > 0 {
6700                job.stem = format!("{}-{}", job.stem, spec.id);
6701            }
6702            let ctx = WaveCtx {
6703                carry_seats: false,
6704                pauses: Some((&self.pause, &self.interrupt)),
6705                run: &run_id,
6706                node,
6707                prompts: &prompts,
6708                cache: cache.as_deref(),
6709                round,
6710            };
6711            let (seat, out) =
6712                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6713            // A resource block never advances the chain: the chosen agent
6714            // was not asked, so it must be the one asked again on resume.
6715            let advances = match &out {
6716                AgentOutcome::Ok(o) => agent::output_advances(o),
6717                AgentOutcome::ResourceBlocked(_) => false,
6718                _ => true,
6719            };
6720            if let Some(next) = ids.get(i + 1)
6721                && advances
6722            {
6723                self.state.seats.insert(seat.key.clone(), seat.clone());
6724                let class =
6725                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6726                let reason = match &out {
6727                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6728                    other => fail_reason(other),
6729                };
6730                // A failed agent's edits must not reach the next one, and a
6731                // restore that did not work ends the chain: the call fails
6732                // and the caller's own (confirmed) restore decides.
6733                if let Some((wt, rev)) = restore
6734                    && let Err(e) = self.graft_restore(wt, rev).await
6735                {
6736                    let why =
6737                        format!("could not restore the worktree before the next fixer: {e:#}");
6738                    tracing::warn!("{why}");
6739                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6740                        AgentOutcome::ResourceBlocked(why)
6741                    } else {
6742                        AgentOutcome::Failed(why)
6743                    };
6744                    last = Some((job, seat, out));
6745                    break;
6746                }
6747                record_handover(
6748                    &mut self.state,
6749                    node,
6750                    &seat.key,
6751                    &spec.id,
6752                    next,
6753                    &class,
6754                    &reason,
6755                );
6756                continue;
6757            }
6758            last = Some((job, seat, out));
6759            break;
6760        }
6761        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6762        if let AgentOutcome::Quota(output) = &out
6763            && node != "graft"
6764        {
6765            // Graft is best effort: its caller records the loss and restores
6766            // the pre-graft tree, then proceeds to review and gate.
6767            // Every fallback was tried. Machine outages cannot be repaired by
6768            // spending a fix round; use each caller's resumable resource path.
6769            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6770            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6771            self.state.quota.push(loss);
6772            self.state.seats.insert(seat.key.clone(), seat.clone());
6773            self.state.status = RunStatus::Blocked;
6774            return (
6775                job,
6776                seat,
6777                AgentOutcome::ResourceBlocked(
6778                    "fixer unavailable; restore agent availability and resume".into(),
6779                ),
6780            );
6781        }
6782        (job, seat, out)
6783    }
6784
6785    /// The one repair a verification-only failure earns (see
6786    /// [`crate::recovery`]), before the first review round of a run that took
6787    /// a branch over.
6788    ///
6789    /// Saved as started *before* the fixer is called, so a resume never
6790    /// repairs twice; a start found on re-entry is rolled back to the commit
6791    /// it began from. A failed or unusable fixer leaves the tree as it was and
6792    /// the review goes on over it, so the first round's own verification
6793    /// reports the failure again. The failure is never carried into a round as
6794    /// a result: verification is re-run on whatever commit this leaves.
6795    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6796        use crate::recovery::{Mode, Repair};
6797        if !self.state.reviews.is_empty() {
6798            return Ok(());
6799        }
6800        let Some(mut rec) = self.state.recovery.clone() else {
6801            return Ok(());
6802        };
6803        if rec.mode != Mode::VerifyRepair {
6804            return Ok(());
6805        }
6806        let wt = winner.worktree.clone();
6807        let mut head = git::rev_parse(&wt, "HEAD").await?;
6808        match rec.repair.clone() {
6809            Repair::Done { .. } => return Ok(()),
6810            Repair::WaitingForRestore { before } => {
6811                self.graft_restore(&wt, &before).await?;
6812                head = before;
6813                rec.repair = Repair::NotStarted;
6814                self.state.recovery = Some(rec.clone());
6815                self.state.save()?;
6816            }
6817            Repair::Started { before } => {
6818                // Interrupted: the unfinished edits are not trusted.
6819                // An unconfirmed rollback blocks the run with `Started` kept,
6820                // so a resume retries it; the tree is never reviewed as is.
6821                if let Err(e) = self.graft_restore(&wt, &before).await {
6822                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6823                        return Err(e);
6824                    }
6825                    return self
6826                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6827                        .await;
6828                }
6829                rec.repair = Repair::Done {
6830                    before: before.clone(),
6831                    after: before,
6832                    committed: false,
6833                    failed: Some("interrupted; rolled back".to_owned()),
6834                };
6835                self.state
6836                    .event("recovery", "the repair was interrupted and rolled back");
6837                self.state.recovery = Some(rec);
6838                self.state.save()?;
6839                return Ok(());
6840            }
6841            Repair::NotStarted => {}
6842        }
6843        if crate::recovery::recheck(&mut rec, &head) {
6844            self.state.event("recovery", rec.summary());
6845            self.state.recovery = Some(rec);
6846            self.state.save()?;
6847            return Ok(());
6848        }
6849        rec.repair = Repair::Started {
6850            before: head.clone(),
6851        };
6852        self.state.recovery = Some(rec.clone());
6853        self.state.save()?;
6854
6855        let language = self.state.config.graph.language.clone();
6856        let sessions = self.state.config.graph.sessions;
6857        let artifacts = agent::artifacts_dir(&self.state.dir());
6858        let fix_prompt = prompt::recovery_fix(
6859            &self.state.instruction,
6860            &rec,
6861            self.state.config.graph.review_rounds,
6862            &language,
6863        );
6864        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6865        self.state.event(
6866            "recovery",
6867            format!(
6868                "repair: the fixer starts from the failed verification on {}",
6869                short(&head)
6870            ),
6871        );
6872        let (_job, seat, out) = self
6873            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6874                prompt: fix_prompt.clone(),
6875                spec,
6876                seat,
6877                cwd: wt.clone(),
6878                timeout,
6879                allow_write: true,
6880                sessions,
6881                artifacts: artifacts.clone(),
6882                stem: "fix-recovery".to_owned(),
6883                handover: None,
6884            })
6885            .await;
6886        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6887            rec.repair = Repair::WaitingForRestore { before: head };
6888            self.state.recovery = Some(rec);
6889            return self.stop_for_resource();
6890        }
6891        let failed = match &out {
6892            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6893            AgentOutcome::Quota(o) => {
6894                self.state.quota.push(QuotaLoss::from_report(
6895                    seat.key.clone(),
6896                    "fix",
6897                    o.quota.as_ref(),
6898                ));
6899                Some("rate limited (quota); the fixer could not run".to_owned())
6900            }
6901            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6902            AgentOutcome::Failed(e) => Some(e.clone()),
6903        };
6904        self.state.seats.insert(seat.key.clone(), seat);
6905        let coordination = self.coordinate(&wt).await?;
6906        let failed = match failed {
6907            // The agent's partial edits are not kept.
6908            Some(why) => {
6909                if let Err(e) = git::restore_to(&wt, &head).await {
6910                    return self
6911                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6912                        .await;
6913                }
6914                Some(why)
6915            }
6916            None => {
6917                if let Ok(r) = git::rescue_commit(
6918                    &wt,
6919                    "magi: recovery repair of the failed verification (uncommitted work)",
6920                )
6921                .await
6922                {
6923                    self.state.note_withheld("fix", &r.withheld);
6924                }
6925                None
6926            }
6927        };
6928        let after = git::rev_parse(&wt, "HEAD").await?;
6929        drop(coordination);
6930        self.state.event(
6931            "recovery",
6932            match &failed {
6933                Some(why) => format!("repair: nothing kept ({why})"),
6934                None if after != head => {
6935                    format!("repair: committed {}", short(&after))
6936                }
6937                None => "repair: the fixer made no commit".to_owned(),
6938            },
6939        );
6940        rec.repair = Repair::Done {
6941            before: head.clone(),
6942            committed: after != head,
6943            after,
6944            failed,
6945        };
6946        self.state.recovery = Some(rec);
6947        self.state.save()?;
6948        Ok(())
6949    }
6950
6951    /// A repair whose rollback could not be confirmed: the run is blocked and
6952    /// resumable, and the unconfirmed state is never reviewed.
6953    async fn stop_recovery(
6954        &mut self,
6955        mut rec: crate::recovery::RecoveryRecord,
6956        head: &str,
6957        why: String,
6958    ) -> Result<()> {
6959        // `Started` stays recorded: a resume retries the restore.
6960        rec.repair = crate::recovery::Repair::Started {
6961            before: head.to_owned(),
6962        };
6963        self.state.recovery = Some(rec);
6964        self.state
6965            .event("recovery", format!("repair could not be undone: {why}"));
6966        self.state.status = RunStatus::Blocked;
6967        self.state.save()?;
6968        bail!("recovery repair could not be rolled back: {why}")
6969    }
6970
6971    async fn review_loop(&mut self) -> Result<()> {
6972        if self.hold_base_sync().await? {
6973            return Ok(());
6974        }
6975        loop {
6976            if self
6977                .state
6978                .base_sync
6979                .as_ref()
6980                .is_some_and(|s| s.conflict.is_some())
6981            {
6982                return Ok(());
6983            }
6984            if self.catchup_repairable() {
6985                let record = self.state.reviews.last().expect("repairable round").clone();
6986                let winner = self
6987                    .state
6988                    .winner()
6989                    .cloned()
6990                    .context("catch-up repair has no winner")?;
6991                if !self.fix_review_round(&winner, record).await? {
6992                    return Ok(());
6993                }
6994            }
6995            self.review_loop_pass().await?;
6996            if !self.catchup_repairable() {
6997                return Ok(());
6998            }
6999        }
7000    }
7001
7002    /// Derive an owed catch-up repair from durable round evidence, including
7003    /// reentry after a crash before the fixer was dispatched.
7004    fn catchup_repairable(&self) -> bool {
7005        self.state.reviews.len() < self.state.config.graph.review_rounds
7006            && self.state.reviews.last().is_some_and(|r| {
7007                !r.clean
7008                    && r.blocking == 0
7009                    && r.fix.is_none()
7010                    && r.e2e_status() == E2eStatus::Failed
7011            })
7012    }
7013
7014    async fn review_loop_pass(&mut self) -> Result<()> {
7015        // A base that would not rebase is a person's decision, not a review
7016        // round: nothing here would change the answer, and reviewers and a
7017        // fixer would be spending real budget on a tree that cannot land
7018        // regardless of what they find.
7019        if self
7020            .state
7021            .base_sync
7022            .as_ref()
7023            .is_some_and(|s| s.conflict.is_some())
7024        {
7025            return Ok(());
7026        }
7027        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7028        // agent files with `magi task add` name the run that paid for it. The
7029        // prompt overlay is cloned alongside it because the waves borrow it
7030        // while `self` is mutably borrowed by the node's own bookkeeping.
7031        let run_id = self.state.id.clone();
7032        let prompts = self.state.config.prompts.clone();
7033        let Some(winner) = self.state.winner().cloned() else {
7034            return Ok(());
7035        };
7036        let granted_review = self
7037            .state
7038            .repair_grants
7039            .last()
7040            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7041            .cloned();
7042        let fresh_review = granted_review
7043            .as_ref()
7044            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7045        let max_rounds = granted_review
7046            .as_ref()
7047            .map_or(self.state.config.graph.review_rounds, |g| {
7048                g.review_start.saturating_add(1)
7049            });
7050        // A clean round, an exhausted round budget, or a stalled tree (see
7051        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7052        // they are recorded — recomputed here, not read off `status`, so a
7053        // reentry into a run that already stopped restates the identical
7054        // verdict instead of silently handing back whatever an earlier node
7055        // in this same walk clobbered `status` to (a solo-candidate
7056        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7057        // below runs an empty range once the budget is spent, and would
7058        // otherwise fall through without touching `status` at all.
7059        if let Some(status) =
7060            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7061        {
7062            // A reentry after a crash between the last round's save and
7063            // `stop_reviewing` reaches the hand-off here, not there.
7064            if status == RunStatus::Gating {
7065                self.record_contested_handoff();
7066            }
7067            self.state.status = status;
7068            self.state.save()?;
7069            return Ok(());
7070        }
7071        self.state.status = RunStatus::Reviewing;
7072        // A last recorded round whose own verification never resolved
7073        // (`ResourceBlocked` — the shared build cache, not the patch) is
7074        // never a concluded round, whatever the round budget says: starting
7075        // a fresh round on top of it would spend a whole new reviewer wave
7076        // re-reading an unchanged patch instead of just retrying the one
7077        // check that actually needs it, and once the budget is spent the
7078        // loop below has nothing left to do at all (its range is empty).
7079        // Retry that check directly instead, exactly the same retry
7080        // `stop_reviewing` already does for its own catch-up case.
7081        if !fresh_review
7082            && self.state.reviews.last().is_some_and(|r| {
7083                r.e2e_status() == E2eStatus::ResourceBlocked
7084                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7085            })
7086        {
7087            let shell = self.state.config.shell();
7088            return self
7089                .stop_unresolved_review(
7090                    "the last round's own verification never resolved",
7091                    &shell,
7092                    &winner.worktree,
7093                )
7094                .await;
7095        }
7096
7097        if granted_review.is_none() {
7098            self.recovery_repair(&winner).await?;
7099        }
7100        if self.resource_halt()? {
7101            return Ok(());
7102        }
7103
7104        let repo = self.state.repo.clone();
7105        let root = self.state.worktree_root();
7106        let language = self.state.config.graph.language.clone();
7107        let sessions = self.state.config.graph.sessions;
7108        let artifacts = self.merge_repair_artifacts();
7109        let base = self.landing_base();
7110        let base_short = short(&base);
7111        let reviewers = self.roles.reviewers.clone();
7112        let shell = self.state.config.shell();
7113
7114        for round in (self.state.reviews.len() + 1)..=max_rounds {
7115            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7116            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7117            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7118            // The prior round's own record, already persisted — never a
7119            // hand-carried variable of just its failing output: that is
7120            // exactly what let a round's e2e result drift out of sync with
7121            // which commit it was actually about (see `SCHEMA`'s doc for
7122            // schema 8). Judged against `head`, the commit reviewers are
7123            // about to look at now, so the summary always reads as "an
7124            // earlier head" here — this round's own patch has not been
7125            // checked yet.
7126            let prev_verification = self
7127                .state
7128                .reviews
7129                .last()
7130                .and_then(|r| r.verification_summary(&head));
7131            // Only the first round of a run that took a branch over: later
7132            // rounds have this run's own history.
7133            let prior_section = match (&self.state.recovery, round) {
7134                (Some(rec), 1) if self.state.reviews.is_empty() => {
7135                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7136                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7137                            Some(crate::recovery::head_of(
7138                                &git::diff(&winner.worktree, reviewed, "HEAD")
7139                                    .await
7140                                    .unwrap_or_default(),
7141                                crate::recovery::SINCE_DIFF_MAX,
7142                            ))
7143                        }
7144                        _ => None,
7145                    };
7146                    Some(prompt::recovery_review(rec, since.as_deref()))
7147                }
7148                _ => None,
7149            };
7150
7151            // Each reviewer gets its own detached checkout of exactly this
7152            // commit: nobody can perturb the winner's tree, and the fixer can
7153            // keep working without racing a reviewer.
7154            let mut jobs = Vec::new();
7155            let coordination = self.coordinate(&repo).await?;
7156            for (r, spec) in reviewers.iter().cloned().enumerate() {
7157                let wt = root.join(format!("review-{}", r + 1));
7158                // `clean -fdx` in a reset wipes setup's products, so every
7159                // round sets the seat up again.
7160                if wt.exists() {
7161                    git::reset_detached(&wt, &head).await?;
7162                } else {
7163                    git::worktree_add_detached(&repo, &wt, &head).await?;
7164                }
7165                self.setup_seat_worktree(&repo, &wt, false).await?;
7166                let seat_key = format!("review-{}", r + 1);
7167                // The seat starts the round on whoever answered it last, not
7168                // on the agent the spec names, so a failure is not re-paid.
7169                let spec = pick_start_spec(
7170                    &self.roles.reviewer_roster,
7171                    spec,
7172                    self.state.seat_history.get(&seat_key),
7173                );
7174                let seat = self.seat(&seat_key, &spec.id);
7175                jobs.push(SeatJob {
7176                    prompt: prompt::review(&prompt::ReviewCtx {
7177                        instruction: &self.state.instruction,
7178                        branch: &winner.branch,
7179                        base_short: &base_short,
7180                        stat: &stat,
7181                        patch: &patch,
7182                        verification: prev_verification.as_ref(),
7183                        prior: prior_section.as_deref(),
7184                        reviewers: reviewers.len(),
7185                        round,
7186                        rounds: max_rounds,
7187                        // A review-only run has no rankings, so nothing
7188                        // competed for this patch and the reviewer is told so.
7189                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7190                        lens: Lens::for_seat(r),
7191                        language: &language,
7192                        github_language: &self.state.config.graph.github_language,
7193                    }),
7194                    spec,
7195                    seat,
7196                    cwd: wt,
7197                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7198                    allow_write: false,
7199                    sessions,
7200                    artifacts: artifacts.clone(),
7201                    stem: format!("review-{round}-{}", r + 1),
7202                    handover: None,
7203                });
7204            }
7205
7206            drop(coordination);
7207            self.state.event(
7208                "review",
7209                format!(
7210                    "round {round}: {} reviewers on {}",
7211                    jobs.len(),
7212                    short(&head)
7213                ),
7214            );
7215            let mut quota_losses = Vec::new();
7216            let review_retries = self.state.config.graph.retries;
7217            let review_cache = self.state.config.cache_dir();
7218            let ctx = WaveCtx {
7219                carry_seats: true,
7220                pauses: Some((&self.pause, &self.interrupt)),
7221                run: &run_id,
7222                node: "review",
7223                prompts: &prompts,
7224                cache: review_cache.as_deref(),
7225                round: Some(round),
7226            };
7227            let results = ask_json_wave::<Review>(
7228                jobs,
7229                Arc::clone(&self.sem),
7230                review_retries,
7231                &self.roles.reviewer_roster,
7232                &ctx,
7233                &mut quota_losses,
7234                &mut self.state,
7235                &|_: &Review| Ok(()),
7236            )
7237            .await;
7238            if self.state.resource_block.is_some() {
7239                return self.stop_for_resource();
7240            }
7241            // Counted before the move below: how many of *this* round's
7242            // reviewer seats were lost to their own rate limit, as opposed to
7243            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7244            let round_quota_missing = quota_losses.len();
7245            self.state.quota.extend(quota_losses);
7246
7247            let mut records = Vec::new();
7248            let mut all_findings = Vec::new();
7249            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7250                let agent_id = seat.agent.clone();
7251                self.state.seats.insert(seat.key.clone(), seat);
7252                let mut record = ReviewRecord {
7253                    reviewer: r + 1,
7254                    agent: agent_id,
7255                    summary: String::new(),
7256                    findings: Vec::new(),
7257                    vote: None,
7258                    failed: None,
7259                    duration_ms: 0,
7260                    // Set for both outcomes: `failed: Some(_)` with
7261                    // `attempts > 0` is a seat every retry still lost, not a
7262                    // recovered one — only `failed: None` with `attempts > 0`
7263                    // reads as "answered after a nudge" (see this field's own
7264                    // doc).
7265                    attempts,
7266                };
7267                match res {
7268                    Ok((review, out)) => {
7269                        // Sanitized here, at the point every other piece of
7270                        // agent prose in this file is (candidate summaries,
7271                        // deliberation turns, vote reasons): a reviewer's own
7272                        // words are the one thing about it that could name
7273                        // it, and reconsideration below broadcasts this same
7274                        // summary and these same findings to every other
7275                        // seat on the panel.
7276                        record.summary =
7277                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7278                        record.vote = Some(review.vote);
7279                        record.duration_ms = out.duration_ms;
7280                        for (n, mut f) in review.findings.into_iter().enumerate() {
7281                            // ids are magi's, never the agent's: the fixer's
7282                            // adoption report is keyed by them.
7283                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7284                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7285                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7286                            // `file` is agent-supplied prose too, never
7287                            // checked against the real tree — the same
7288                            // exposure `title`/`detail` above have, just in
7289                            // a field easy to forget because it looks like a
7290                            // path rather than free text.
7291                            f.file = f
7292                                .file
7293                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7294                            all_findings.push(f.clone());
7295                            record.findings.push(f);
7296                        }
7297                        self.state.event(
7298                            "review",
7299                            format!(
7300                                "round {round}: reviewer {} voted {} with {} finding(s)",
7301                                r + 1,
7302                                review.vote.label(),
7303                                record.findings.len()
7304                            ),
7305                        );
7306                    }
7307                    Err(e) => {
7308                        record.failed = Some(e.to_string());
7309                        self.state.event(
7310                            "review",
7311                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7312                        );
7313                    }
7314                }
7315                records.push(record);
7316            }
7317
7318            // Tally the round's votes and, if they split, spend the one
7319            // round of reconsideration the split -> deliberate -> revote
7320            // shape `judge`/`vote` use for the panel, sized down to what a
7321            // read-only review round can afford: one round, and a revote
7322            // rather than an argument, because the panel already wrote its
7323            // reasoning down as findings the first time around.
7324            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7325            let vote_split =
7326                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7327            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7328            if vote_split {
7329                self.state.event(
7330                    "review",
7331                    format!(
7332                        "round {round}: votes split ({}) — one round of reconsideration",
7333                        initial_votes
7334                            .iter()
7335                            .map(|v| v.label())
7336                            .collect::<Vec<_>>()
7337                            .join(", ")
7338                    ),
7339                );
7340                // Seats read every seat's findings and votes, still numbered
7341                // and never named — the same anonymity `review` itself keeps.
7342                let panel: Vec<ReviewSeatReport<'_>> = records
7343                    .iter()
7344                    .filter_map(|r| {
7345                        r.vote.map(|vote| ReviewSeatReport {
7346                            reviewer: r.reviewer,
7347                            vote,
7348                            summary: &r.summary,
7349                            findings: &r.findings,
7350                        })
7351                    })
7352                    .collect();
7353
7354                let mut jobs = Vec::new();
7355                let mut seats_at = Vec::new();
7356                for (r, spec) in reviewers.iter().cloned().enumerate() {
7357                    // A seat with no initial vote has nothing to reconsider
7358                    // from and stays absent, the same as it stayed absent
7359                    // from `panel` above.
7360                    if records[r].vote.is_none() {
7361                        continue;
7362                    }
7363                    let wt = root.join(format!("review-{}", r + 1));
7364                    let seat_key = format!("review-{}", r + 1);
7365                    let spec = self.occupant(&seat_key, spec);
7366                    let seat = self.seat(&seat_key, &spec.id);
7367                    // A seat with no live session has already forgotten the
7368                    // initial review's prompt — restate the patch it is
7369                    // voting on, the same as `deliberate`/`vote` do for a
7370                    // judge in the same position.
7371                    // The panel already carries this seat's own review and
7372                    // vote, so restating the patch makes the prompt whole for
7373                    // a seat handed to another agent.
7374                    let build = |with_patch: bool| {
7375                        prompt::review_reconsider(&ReviewReconsiderCtx {
7376                            instruction: &self.state.instruction,
7377                            reviewer: r + 1,
7378                            lens: Lens::for_seat(r),
7379                            panel: &panel,
7380                            patch: with_patch.then_some(ReviewPatch {
7381                                branch: &winner.branch,
7382                                base_short: &base_short,
7383                                stat: &stat,
7384                                patch: &patch,
7385                            }),
7386                            round,
7387                            rounds: max_rounds,
7388                            language: &language,
7389                        })
7390                    };
7391                    let full = build(true);
7392                    let prompt = if has_context(&spec, &seat, sessions) {
7393                        build(false)
7394                    } else {
7395                        full.clone()
7396                    };
7397                    jobs.push(SeatJob {
7398                        prompt,
7399                        spec,
7400                        seat,
7401                        cwd: wt,
7402                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7403                        allow_write: false,
7404                        sessions,
7405                        artifacts: artifacts.clone(),
7406                        stem: format!("review-{round}-reconsider-{}", r + 1),
7407                        handover: Some(full),
7408                    });
7409                    seats_at.push(r);
7410                }
7411
7412                let mut recon_quota_losses = Vec::new();
7413                let recon_cache = self.state.config.cache_dir();
7414                let recon_ctx = WaveCtx {
7415                    carry_seats: true,
7416                    pauses: Some((&self.pause, &self.interrupt)),
7417                    run: &run_id,
7418                    node: "review",
7419                    prompts: &prompts,
7420                    cache: recon_cache.as_deref(),
7421                    round: Some(round),
7422                };
7423                let recon_results = ask_json_wave::<ReviewRevote>(
7424                    jobs,
7425                    Arc::clone(&self.sem),
7426                    review_retries,
7427                    &self.roles.reviewer_roster,
7428                    &recon_ctx,
7429                    &mut recon_quota_losses,
7430                    &mut self.state,
7431                    &|_: &ReviewRevote| Ok(()),
7432                )
7433                .await;
7434                if self.state.resource_block.is_some() {
7435                    return self.stop_for_resource();
7436                }
7437                self.state.quota.extend(recon_quota_losses);
7438
7439                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7440                    let agent_id = seat.agent.clone();
7441                    self.state.seats.insert(seat.key.clone(), seat);
7442                    let mut rec = ReviewRevoteRecord {
7443                        reviewer: r + 1,
7444                        agent: agent_id,
7445                        vote: None,
7446                        reason: String::new(),
7447                        failed: None,
7448                    };
7449                    match res {
7450                        Ok((rv, _)) => {
7451                            rec.vote = Some(rv.vote);
7452                            rec.reason =
7453                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7454                            self.state.event(
7455                                "review",
7456                                format!(
7457                                    "round {round}: reviewer {} revoted {}",
7458                                    r + 1,
7459                                    rv.vote.label()
7460                                ),
7461                            );
7462                        }
7463                        Err(e) => {
7464                            rec.failed = Some(e.to_string());
7465                            self.state.event(
7466                                "review",
7467                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7468                            );
7469                        }
7470                    }
7471                    reconsideration.push(rec);
7472                }
7473            } else if initial_votes.len() > 1 {
7474                self.state.event(
7475                    "review",
7476                    format!(
7477                        "round {round}: votes agreed ({}) — no reconsideration",
7478                        initial_votes[0].label()
7479                    ),
7480                );
7481            }
7482
7483            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7484            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7485            // A round that already has a blocking finding and a round left to
7486            // try is going back to the fixer no matter what `verify.e2e`
7487            // says, so running it first only spends the loop's slowest step
7488            // (minutes, for a Rust repo's full test suite) on a head about
7489            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7490            // runs once a round has no blocking findings left (see
7491            // `round_is_clean`, which a deferred — empty — `e2e` can never
7492            // satisfy since `blocking` is nonzero whenever this branch is
7493            // taken), and `stop_reviewing` forces a real run before it will
7494            // ever read a deferred round as green.
7495            let defer_e2e =
7496                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7497            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7498                let reason =
7499                    format!("{blocking} blocking finding(s) already required a fix this round");
7500                self.state.event(
7501                    "verify",
7502                    format!(
7503                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7504                         {}); it will run once a round has none left",
7505                        short(&head)
7506                    ),
7507                );
7508                (Vec::new(), false, true, Some(reason))
7509            } else {
7510                let e2e_commands = self.state.config.verify.e2e.clone();
7511                let cache_dir = self.state.config.cache_dir();
7512                let context = format!("round {round}");
7513                let wait = self.repository_wait();
7514                let (e2e, verify_retried) = with_cache_lease(
7515                    &mut self.state,
7516                    cache_dir.as_deref(),
7517                    "e2e",
7518                    "e2e",
7519                    &winner.worktree,
7520                    &head,
7521                    verify_timeout,
7522                    &context,
7523                    &wait,
7524                    |state, budget| {
7525                        let shell = shell.clone();
7526                        let e2e_commands = e2e_commands.clone();
7527                        let worktree = winner.worktree.clone();
7528                        let context = context.clone();
7529                        async move {
7530                            run_e2e_with_retry(
7531                                state,
7532                                &shell,
7533                                &e2e_commands,
7534                                &worktree,
7535                                budget,
7536                                &context,
7537                            )
7538                            .await
7539                        }
7540                    },
7541                )
7542                .await?;
7543                (e2e, verify_retried, false, None)
7544            };
7545
7546            let expected = records.len();
7547            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7548            let incomplete = answered < expected;
7549            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7550            let policy = self.state.config.graph.incomplete_review;
7551            let clean = round_is_clean(
7552                blocking,
7553                e2e_ok,
7554                answered,
7555                expected,
7556                round_quota_missing,
7557                policy,
7558            );
7559
7560            let mut round_record = ReviewRound {
7561                round,
7562                head: head.clone(),
7563                verified_head: None,
7564                verified_at: None,
7565                reviews: records,
7566                e2e,
7567                verify_retried,
7568                e2e_deferred,
7569                e2e_defer_reason,
7570                fix: None,
7571                blocking,
7572                answered,
7573                expected,
7574                clean,
7575                progressed: false,
7576                vote_split,
7577                reconsideration,
7578                verdict: None,
7579            };
7580            // The final vote per seat is its revote where reconsideration
7581            // ran and answered, its initial vote otherwise — the same
7582            // fallback `tally` uses for a judge whose private vote failed.
7583            round_record.verdict = ReviewVote::worst(
7584                round_record
7585                    .final_votes()
7586                    .into_iter()
7587                    .map(|(_, _, vote)| vote),
7588            );
7589            // Which commit and when magi actually attempted to check —
7590            // known the moment a command was dispatched against `head`,
7591            // whether or not it finished: a resource-blocked attempt still
7592            // targeted a specific commit at a specific time, and leaving
7593            // that unrecorded is exactly what made `verification_summary`
7594            // report a fresh attempt as "commit unknown ... recorded before
7595            // this was tracked", indistinguishable from a genuinely old,
7596            // untracked record. Only a deferred or unconfigured round never
7597            // ran at all and has nothing to record — see
7598            // `ReviewRound::verified_head`'s own doc.
7599            if !matches!(
7600                round_record.e2e_status(),
7601                E2eStatus::Deferred | E2eStatus::NotConfigured
7602            ) {
7603                round_record.verified_head = Some(head.clone());
7604                round_record.verified_at = Some(Timestamp::now());
7605            }
7606
7607            if incomplete {
7608                let missing: Vec<String> = round_record
7609                    .reviews
7610                    .iter()
7611                    .filter(|r| r.failed.is_some())
7612                    .map(|r| format!("review-{}", r.reviewer))
7613                    .collect();
7614                self.state.event(
7615                    "review",
7616                    format!(
7617                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7618                        missing.join(", ")
7619                    ),
7620                );
7621            }
7622
7623            if clean {
7624                self.state.event(
7625                    "review",
7626                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7627                        format!(
7628                            "round {round}: clean (warn policy, incomplete panel) — no \
7629                             blocking findings from the seats that answered, verification green"
7630                        )
7631                    } else if incomplete {
7632                        format!(
7633                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7634                             quorum) — no blocking findings from the seats that answered, \
7635                             verification green",
7636                            expected - answered
7637                        )
7638                    } else {
7639                        format!("round {round}: clean — no blocking findings, verification green")
7640                    },
7641                );
7642                self.state.reviews.push(round_record);
7643                self.state.status = RunStatus::Gating;
7644                self.state.save()?;
7645                return Ok(());
7646            }
7647
7648            // Nothing was raised and verification passed, but not every seat
7649            // answered and `round_is_clean` still refused to call it clean —
7650            // either a seat is missing for a reason other than its own quota
7651            // (a crash, a timeout, unparsable output — worth another try), or
7652            // every seat that could have answered lost its quota and nobody
7653            // is left to decide on: re-review rather than send the fixer
7654            // after a round with nothing to fix.
7655            if incomplete && blocking == 0 && e2e_ok {
7656                self.state.reviews.push(round_record);
7657                self.state.save()?;
7658                if round == max_rounds {
7659                    self.state.status = RunStatus::Blocked;
7660                    self.state.event(
7661                        "review",
7662                        format!(
7663                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7664                             refusing to call it clean",
7665                            expected - answered
7666                        ),
7667                    );
7668                    return Ok(());
7669                }
7670                continue;
7671            }
7672
7673            // Nothing for the fixer to act on (`blocking == 0`) and the only
7674            // reason this round is not clean is that magi itself never got
7675            // a command to run — the shared build cache, not the patch (see
7676            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7677            // the fixer would invite a change to appease contention that has
7678            // nothing to do with the diff, and would leave this attempt
7679            // sitting in the next round's prompt as if it were about an
7680            // earlier, superseded commit rather than what it actually is:
7681            // the same head, still waiting to be checked. Wait for it the
7682            // same way the final round's own contention is already handled,
7683            // whatever round this happens to be.
7684            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7685                self.state.reviews.push(round_record);
7686                return self
7687                    .stop_unresolved_review(
7688                        "the round's own verification could not run",
7689                        &shell,
7690                        &winner.worktree,
7691                    )
7692                    .await;
7693            }
7694
7695            if round == max_rounds {
7696                self.state.reviews.push(round_record);
7697                return self
7698                    .stop_reviewing(
7699                        &format!(
7700                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7701                        ),
7702                        &shell,
7703                        &winner.worktree,
7704                    )
7705                    .await;
7706            }
7707
7708            if !self.fix_review_round(&winner, round_record).await? {
7709                return Ok(());
7710            }
7711        }
7712        Ok(())
7713    }
7714
7715    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7716    /// False means admission or continuation policy stopped the review loop.
7717    async fn fix_review_round(
7718        &mut self,
7719        winner: &Candidate,
7720        mut round_record: ReviewRound,
7721    ) -> Result<bool> {
7722        let round = round_record.round;
7723        let shell = self.state.config.shell();
7724        let step = self.run_fix_round(winner, &mut round_record).await?;
7725        if matches!(step, FixStep::Resource) {
7726            self.stop_for_resource()?;
7727            return Ok(false);
7728        }
7729        if self.state.reviews.len() < round {
7730            self.state.reviews.push(round_record);
7731        } else {
7732            self.state.reviews[round - 1] = round_record;
7733        }
7734        self.state.save()?;
7735
7736        // The fixer's own report never came back this round, even after
7737        // `continue_fix_report`'s own budget was spent on it — not an
7738        // ordinary "no report" (dropped stream, quota, plain failure),
7739        // which already reads that way and is left to the existing round
7740        // budget. Stopping here, rather than opening another round, is
7741        // what keeps a next reviewer/fixer wave from ever being
7742        // dispatched onto `winner.worktree` while whatever the seat's
7743        // last call may still have running there is unaccounted for: no
7744        // process liveness check exists (and none is being added — see
7745        // AGENTS.md/this task's own scope), so the only way to honour
7746        // "nothing starts before a valid report returns" is to not start
7747        // anything further on this worktree from this run at all.
7748        if matches!(step, FixStep::Recorded { report_lost: true }) {
7749            self.stop_reviewing(
7750                "the fixer's adoption report never came back, even after resuming its \
7751                         own seat; refusing to start another round against the same worktree \
7752                         while that is unresolved",
7753                &shell,
7754                &winner.worktree,
7755            )
7756            .await?;
7757            return Ok(false);
7758        }
7759
7760        let streak = self
7761            .state
7762            .reviews
7763            .iter()
7764            .rev()
7765            .take_while(|r| !r.progressed)
7766            .count();
7767        if streak >= STAGNANT_LIMIT {
7768            self.stop_reviewing(
7769                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7770                &shell,
7771                &winner.worktree,
7772            )
7773            .await?;
7774            return Ok(false);
7775        }
7776        Ok(true)
7777    }
7778
7779    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7780    async fn run_fix_round(
7781        &mut self,
7782        winner: &Candidate,
7783        round_record: &mut ReviewRound,
7784    ) -> Result<FixStep> {
7785        let round = round_record.round;
7786        let max_rounds = self.state.config.graph.review_rounds;
7787        let run_id = self.state.id.clone();
7788        let prompts = self.state.config.prompts.clone();
7789        let language = self.state.config.graph.language.clone();
7790        let sessions = self.state.config.graph.sessions;
7791        let artifacts = self.merge_repair_artifacts();
7792        let base = self.landing_base();
7793        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7794        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7795        let this_round_verification = round_record.verification_summary(&head);
7796        let all_findings: Vec<_> = round_record
7797            .reviews
7798            .iter()
7799            .flat_map(|r| r.findings.iter().cloned())
7800            .collect();
7801        // Fix. The winner's own implementer seat continues its conversation:
7802        // the competition is over, so context is pure benefit now.
7803        let blocking_findings: Vec<_> = all_findings
7804            .iter()
7805            .filter(|f| f.severity.blocks())
7806            .cloned()
7807            .collect();
7808        let fix_prompt = prompt::fix_in(
7809            &self.state.instruction,
7810            &blocking_findings,
7811            this_round_verification.as_ref(),
7812            round,
7813            max_rounds,
7814            &language,
7815            &self.state.config.graph.github_language,
7816        );
7817        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7818        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7819        let (job, seat, out) = self
7820            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7821                prompt: fix_prompt.clone(),
7822                spec,
7823                seat,
7824                cwd: winner.worktree.clone(),
7825                timeout,
7826                allow_write: true,
7827                sessions,
7828                artifacts: artifacts.clone(),
7829                stem: format!("fix-{round}"),
7830                handover: None,
7831            })
7832            .await;
7833        // Not asked: the round is not recorded (so a resume repeats it),
7834        // no fixer was handed over to, and the seat is left as it was.
7835        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7836            return Ok(FixStep::Resource);
7837        }
7838        let agent_id = seat.agent.clone();
7839
7840        let mut fix = FixRecord {
7841            agent: agent_id,
7842            addressed: Vec::new(),
7843            rejected: Vec::new(),
7844            notes: String::new(),
7845            committed: false,
7846            failed: None,
7847            duration_ms: 0,
7848            continuation: None,
7849        };
7850        let mut continuation = ContinuationRecord::not_needed();
7851        let mut final_seat = seat.clone();
7852        match out {
7853            AgentOutcome::Ok(o) => {
7854                fix.duration_ms = o.duration_ms;
7855                let parsed = verdict::extract_json::<FixReport>(&o.text);
7856                // A parsed report standing next to a command this same
7857                // reply's own CLI never confirmed the exit status of is
7858                // not a resolved answer — the identical `CommandEvidence`
7859                // `state.jobs` renders, read here instead of only on
7860                // display, per the completion judgment and the shown
7861                // record needing to agree.
7862                let incomplete_reason = match &parsed {
7863                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7864                        "the reply parsed, but it reported a command whose own CLI never \
7865                             confirmed an exit status"
7866                            .to_owned(),
7867                    ),
7868                    Ok(_) => None,
7869                    Err(e) => Some(e.to_string()),
7870                };
7871                match incomplete_reason {
7872                    None => {
7873                        let report = parsed.expect("checked Ok above");
7874                        fix.addressed = report.addressed;
7875                        fix.rejected = report.rejected;
7876                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7877                    }
7878                    Some(reason) => {
7879                        let (resumed_seat, resolved, failure, cont) = self
7880                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7881                            .await;
7882                        fix.duration_ms += cont.cumulative_wait_ms;
7883                        continuation = cont;
7884                        final_seat = resumed_seat;
7885                        match resolved {
7886                            Some(report) => {
7887                                fix.addressed = report.addressed;
7888                                fix.rejected = report.rejected;
7889                                fix.notes =
7890                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7891                            }
7892                            None => fix.failed = failure,
7893                        }
7894                    }
7895                }
7896            }
7897            // The CLI's raw error JSON is not a fix report to parse.
7898            AgentOutcome::Dropped(o) => {
7899                fix.duration_ms = o.duration_ms;
7900                let why = o
7901                    .dropped
7902                    .as_ref()
7903                    .map(|d| d.why.as_str())
7904                    .unwrap_or("the CLI ended the stream without delivering its answer");
7905                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7906            }
7907            AgentOutcome::Quota(o) => {
7908                self.state.quota.push(QuotaLoss::from_report(
7909                    final_seat.key.clone(),
7910                    "fix",
7911                    o.quota.as_ref(),
7912                ));
7913                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7914            }
7915            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7916        }
7917        if self.state.resource_block.is_some() {
7918            self.state.seats.insert(final_seat.key.clone(), final_seat);
7919            return Ok(FixStep::Resource);
7920        }
7921        fix.continuation = Some(continuation);
7922        self.state.seats.insert(final_seat.key.clone(), final_seat);
7923        let coordination = self.coordinate(&winner.worktree).await?;
7924        if let Ok(r) = git::rescue_commit(
7925            &winner.worktree,
7926            &format!("magi: review round {round} fixes (uncommitted work)"),
7927        )
7928        .await
7929        {
7930            self.state.note_withheld("fix", &r.withheld);
7931        }
7932        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7933        fix.committed = after != before;
7934        // Judged by what `git` says moved against base, never by the
7935        // fixer's own `addressed`/`rejected` count — see
7936        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7937        // `patch` snapshot above: swallowing this error would default
7938        // `diff_after` to empty, which almost always differs from a
7939        // non-empty `patch` and reads as "progressed" — exactly backwards
7940        // for a `git` failure the stagnation check cannot see through.
7941        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7942        drop(coordination);
7943        let progressed = diff_after != patch;
7944        let commit_note = if fix.committed {
7945            "committed"
7946        } else {
7947            "NO new commit"
7948        };
7949        let tree_note = if progressed {
7950            "changed vs base"
7951        } else {
7952            "unchanged vs base"
7953        };
7954        self.state.event(
7955            "fix",
7956            match &fix.failed {
7957                // Distinct on purpose from "0 addressed, 0 rejected": the
7958                // fixer's own diff still landed (blocking counts do keep
7959                // falling round over round), only its adoption report did
7960                // not come back, so this must never read like every
7961                // finding was reviewed and declined.
7962                Some(reason) => {
7963                    format!(
7964                        "round {round}: fixer's adoption report was lost ({reason}); \
7965                             {commit_note}, tree {tree_note}"
7966                    )
7967                }
7968                None => format!(
7969                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7970                         {tree_note}{}",
7971                    fix.addressed.len(),
7972                    fix.rejected.len(),
7973                    if continuation.outcome == ContinuationOutcome::Resumed {
7974                        format!(
7975                            " (adoption report recovered after {} continuation(s))",
7976                            continuation.attempts
7977                        )
7978                    } else {
7979                        String::new()
7980                    },
7981                ),
7982            },
7983        );
7984        round_record.fix = Some(fix);
7985        round_record.progressed = progressed;
7986        Ok(FixStep::Recorded {
7987            report_lost: matches!(
7988                continuation.outcome,
7989                ContinuationOutcome::Exhausted
7990                    | ContinuationOutcome::QuotaLost
7991                    | ContinuationOutcome::NoSession
7992            ),
7993        })
7994    }
7995
7996    /// Decide, from the last recorded round's own verification, whether
7997    /// stopping the review loop is a hand-off or a genuine block.
7998    ///
7999    /// Called once the loop has given up trying — the round budget is spent,
8000    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8001    /// findings still open, never while a round is still clean or the
8002    /// incomplete-panel case handled inline above. Gate and e2e are facts
8003    /// about the tree; a lingering review finding is an opinion, and this
8004    /// workload's own `magi stats` puts reviewer precision low enough
8005    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8006    /// must not by itself stand between a green, verified change and the
8007    /// human who decides what to do with it. A red e2e is not an opinion, so
8008    /// that case still blocks, with the failing command and a tail of its
8009    /// output recorded here rather than left in `run.json` for someone to go
8010    /// find.
8011    ///
8012    /// A round that deferred its own e2e (see [`Config::graph`]'s
8013    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8014    /// only because nothing ran, and treating an empty list as a passing one
8015    /// here is exactly the "deferred painted green" bug this function exists
8016    /// to not have. When the last round's own verification never resolved —
8017    /// deferred on purpose, or a real attempt the shared build cache blocked
8018    /// — this makes (or retries) the real run, on the actual worktree this
8019    /// loop is about to stop touching, before deciding anything. A
8020    /// resource-blocked attempt is likewise never read as either green or
8021    /// red: it is evidence about the machine, not the patch (see
8022    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8023    /// blocked cache leaves this call without deciding rather than guessing
8024    /// — the caller retries on a later reentry.
8025    /// Record, once, that the review loop handed off over a blocking finding
8026    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8027    /// `land` asks the owner even with `land_approval` off. Called from every
8028    /// path that concludes `Gating`; a reentry keeps the first record.
8029    fn record_contested_handoff(&mut self) {
8030        if self.state.contested_handoff.is_some() {
8031            return;
8032        }
8033        let Some(contested) = self
8034            .state
8035            .reviews
8036            .last()
8037            .and_then(ReviewRound::contested_handoff)
8038        else {
8039            return;
8040        };
8041        self.state.event(
8042            "review",
8043            format!(
8044                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8045                 wait for the owner's approval",
8046                contested.findings.len(),
8047                contested.rejecters.len()
8048            ),
8049        );
8050        self.state.contested_handoff = Some(contested);
8051    }
8052
8053    async fn stop_unresolved_review(
8054        &mut self,
8055        why: &str,
8056        shell: &[String],
8057        worktree: &Path,
8058    ) -> Result<()> {
8059        self.stop_reviewing_inner(why, shell, worktree, true).await
8060    }
8061
8062    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8063        self.stop_reviewing_inner(why, shell, worktree, false).await
8064    }
8065
8066    async fn stop_reviewing_inner(
8067        &mut self,
8068        why: &str,
8069        shell: &[String],
8070        worktree: &Path,
8071        repairable: bool,
8072    ) -> Result<()> {
8073        let round_idx = self.state.reviews.len() - 1;
8074        // A deferred round and a resource-blocked one are the same shape
8075        // here: neither has a real result yet, and both get one more
8076        // attempt. Read off `e2e_status` — the single source for this —
8077        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8078        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8079        // still retries instead of being read as a settled result the
8080        // instant it stops being empty.
8081        let needs_catchup_run = matches!(
8082            self.state.reviews[round_idx].e2e_status(),
8083            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8084        );
8085        if needs_catchup_run {
8086            let round = self.state.reviews[round_idx].round;
8087            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8088            let commands = self.state.config.verify.e2e.clone();
8089            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8090            let cache_dir = self.state.config.cache_dir();
8091            let context = format!(
8092                "round {round}: verification unresolved, catching up before the final decision"
8093            );
8094            let wait = self.repository_wait();
8095            let (outcomes, verify_retried) = with_cache_lease(
8096                &mut self.state,
8097                cache_dir.as_deref(),
8098                "e2e",
8099                "e2e",
8100                worktree,
8101                &attempted_head,
8102                timeout,
8103                &context,
8104                &wait,
8105                |state, budget| {
8106                    let shell = shell.to_vec();
8107                    let commands = commands.clone();
8108                    let context = context.clone();
8109                    async move {
8110                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8111                            .await
8112                    }
8113                },
8114            )
8115            .await?;
8116            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8117                self.state.record_base_progress(&attempted_head);
8118            }
8119            let last = &mut self.state.reviews[round_idx];
8120            last.e2e = outcomes;
8121            last.verify_retried = verify_retried;
8122            // Always the commit and time this attempt actually targeted,
8123            // whether or not it happens to equal the reviewed `head` and
8124            // whether or not a command finished — see
8125            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8126            // attempt is recorded too, so a later reader sees "attempted
8127            // again at T2" rather than silence.
8128            last.verified_head = Some(attempted_head);
8129            last.verified_at = Some(Timestamp::now());
8130            if verify_inconclusive(&last.e2e) {
8131                // Still not a real result: `e2e_deferred` is left exactly
8132                // as it was, so `needs_catchup_run` above reads
8133                // `ResourceBlocked` (via `e2e_status`, which checks
8134                // `resource_blocked` before `e2e_deferred`) and retries
8135                // again on the next reentry, rather than recording
8136                // contention as a red e2e and blocking the run on it.
8137                self.state.save()?;
8138                return Ok(());
8139            }
8140            last.e2e_deferred = false;
8141            // A complete panel may have been saved as unclean solely because
8142            // verification could not run. Refresh that verdict with the new
8143            // outcomes before a granted repair consumes the round's result.
8144            // Incomplete panels retain their original policy/quota decision.
8145            if last.answered == last.expected {
8146                last.clean = round_is_clean(
8147                    last.blocking,
8148                    last.e2e.iter().all(CommandOutcome::ok),
8149                    last.answered,
8150                    last.expected,
8151                    0,
8152                    self.state.config.graph.incomplete_review,
8153                );
8154            }
8155        }
8156        let last = &self.state.reviews[round_idx];
8157        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8158
8159        match last.e2e_status() {
8160            E2eStatus::Failed => {
8161                let red: Vec<String> = last
8162                    .e2e
8163                    .iter()
8164                    .filter(|o| !o.ok())
8165                    .map(|o| {
8166                        format!(
8167                            "`{}` -> {:?}\n{}",
8168                            o.command,
8169                            o.code,
8170                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8171                        )
8172                    })
8173                    .collect();
8174                let can_repair = last.blocking == 0 && last.fix.is_none();
8175                self.state.reviews[round_idx].clean = false;
8176                self.state
8177                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8178                self.state.status = RunStatus::Blocked;
8179                if repairable
8180                    && can_repair
8181                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8182                {
8183                    self.state.status = RunStatus::Reviewing;
8184                }
8185            }
8186            // `needs_catchup_run` above already retried once this call; if
8187            // it is still blocked, this is magi's own admission it could
8188            // not get a command to run, never a verdict on the patch — the
8189            // run is left exactly where a later reentry can retry again.
8190            E2eStatus::ResourceBlocked => {
8191                self.state.event(
8192                    "review",
8193                    format!(
8194                        "{why}; e2e could not run (resource unavailable); not \
8195                         deciding yet"
8196                    ),
8197                );
8198            }
8199            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8200                self.state.event(
8201                    "review",
8202                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8203                );
8204                self.record_contested_handoff();
8205                self.state.status = RunStatus::Gating;
8206            }
8207        }
8208        self.state.save()?;
8209        Ok(())
8210    }
8211
8212    // ----------------------------------------------------------------- gate
8213
8214    async fn gate(&mut self) -> Result<()> {
8215        if self.hold_base_sync().await? {
8216            return Ok(());
8217        }
8218        // Judged by the review record itself, not by `status`: a solo
8219        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8220        // reentry (see `judge`), and trusting it here is exactly how a run
8221        // that exhausted its review budget got gated and merged a second
8222        // time around. `review_conclusion` recomputes the review loop's own
8223        // verdict from the round records themselves — `Gating` for a clean
8224        // round or a hand-off (see `stop_reviewing`), anything else means the
8225        // loop is still going or genuinely blocked.
8226        // A base the winner could not be replayed onto is a decision, not a
8227        // round: there is no landing tree to gate. Read as its own record for
8228        // the same reason the review verdict is.
8229        if self.state.status == RunStatus::Failed
8230            || self
8231                .state
8232                .base_sync
8233                .as_ref()
8234                .is_some_and(|s| s.conflict.is_some())
8235            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8236                != Some(RunStatus::Gating)
8237        {
8238            return Ok(());
8239        }
8240        let Some(winner) = self.state.winner().cloned() else {
8241            return Ok(());
8242        };
8243        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8244        self.state.reconcile_verification_head(&head);
8245        self.state.save()?;
8246        let wait = self.repository_wait();
8247        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8248            return Ok(());
8249        }
8250        if self.state.gate_ran {
8251            // `review_loop` derives its conclusion from the clean review
8252            // record on every reentry and therefore puts a completed run back
8253            // in `Gating`. A recorded gate is a stronger, terminal fact:
8254            // retain its original command output (or lack of any, for a repo
8255            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8256            // doc) and restore `Blocked` on a real failure rather than
8257            // pretending the command is still running or running it a second
8258            // time. `gate_ran == false` remains the only shape — unattempted,
8259            // or a resource-blocked retry — that may still need to execute a
8260            // command.
8261            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8262                self.state.status = RunStatus::Blocked;
8263                self.state.save()?;
8264            }
8265            return Ok(());
8266        }
8267        self.state.status = RunStatus::Gating;
8268        let mut outcomes = self.run_gate(&winner).await?;
8269        let mut attempted_head = self.state.gate_head.clone();
8270        loop {
8271            // A resource-blocked outcome means the gate command never actually
8272            // ran - the shared build cache could not be acquired or confirmed
8273            // fresh in time - which is evidence about the machine, not about
8274            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8275            // Recording it as a red gate would mark a run `Blocked` on nothing
8276            // but contention magi has already logged; leaving `self.state.gate`
8277            // empty and `self.state.gate_ran` false instead keeps the shape
8278            // this function already treats as "still needs to run" (see the
8279            // early-return above), so the next call retries the command
8280            // rather than concluding anything.
8281            if verify_inconclusive(&outcomes) {
8282                self.state.save()?;
8283                return Ok(());
8284            }
8285            if outcomes.iter().all(CommandOutcome::ok) {
8286                break;
8287            }
8288            let fixes_before = self.state.gate_fixes.len();
8289            match self.gate_fix_round(&winner, &outcomes).await? {
8290                GateFix::Retry => {
8291                    outcomes = self.run_gate(&winner).await?;
8292                    attempted_head = self.state.gate_head.clone();
8293                }
8294                GateFix::Stop => {
8295                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8296                    if self.state.gate_fixes.len() > fixes_before
8297                        && attempted_head.as_deref() != Some(current.as_str())
8298                        && verify_current_head(&mut self.state, &winner, &wait).await?
8299                    {
8300                        // A new SHA owes a gate even when a spent fixer changed only
8301                        // commit metadata. This does not grant another repair round.
8302                        outcomes = self.run_gate(&winner).await?;
8303                        attempted_head = self.state.gate_head.clone();
8304                    } else {
8305                        break;
8306                    }
8307                }
8308                GateFix::Defer => {
8309                    self.state.save()?;
8310                    return Ok(());
8311                }
8312            }
8313        }
8314        let passed = outcomes.iter().all(CommandOutcome::ok);
8315        self.state.gate = outcomes;
8316        self.state.gate_ran = true;
8317        self.state.gate_head = attempted_head;
8318        if !self.state.gate.is_empty()
8319            && !verify_inconclusive(&self.state.gate)
8320            && let Some(head) = self.state.gate_head.clone()
8321        {
8322            self.state.record_base_progress(&head);
8323        }
8324        self.state.save()?;
8325        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8326            return Ok(());
8327        }
8328        if !passed {
8329            self.state.status = RunStatus::Blocked;
8330            let spent = self.state.gate_fixes.len();
8331            self.state.event(
8332                "gate",
8333                if spent == 0 {
8334                    "gate failed; not merging".to_owned()
8335                } else {
8336                    format!("gate failed after {spent} gate-fix round(s); not merging")
8337                },
8338            );
8339        }
8340        self.state.save()?;
8341        Ok(())
8342    }
8343
8344    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8345    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8346    /// after review is clean and never on a candidate awaiting judging.
8347    ///
8348    /// Never fails the run: a non-zero exit or timeout is a warning and a
8349    /// recorded outcome, and the gate remains the single arbiter. Nothing
8350    /// configured means nothing happens - no event, no commit. `commit_all`
8351    /// commits any leftover change under the neutral identity and returns
8352    /// `false` when the tree is clean, so no empty commit is ever made.
8353    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8354        let commands = state.config.verify.pre_gate.clone();
8355        if commands.is_empty() {
8356            return;
8357        }
8358        let shell = state.config.shell();
8359        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8360        let (outcomes, _) = run_commands(
8361            state,
8362            "pre_gate",
8363            "pre_gate",
8364            0,
8365            &shell,
8366            &commands,
8367            &winner.worktree,
8368            timeout,
8369        )
8370        .await;
8371        for o in &outcomes {
8372            if !o.ok() {
8373                tracing::warn!(
8374                    "pre_gate `{}` failed ({:?}); the gate decides",
8375                    o.command,
8376                    o.code
8377                );
8378            }
8379            state.event(
8380                "pre_gate",
8381                format!(
8382                    "`{}` -> {}",
8383                    o.command,
8384                    if o.ok() {
8385                        "pass".to_owned()
8386                    } else {
8387                        format!(
8388                            "FAIL ({:?})\n{}",
8389                            o.code,
8390                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8391                        )
8392                    }
8393                ),
8394            );
8395        }
8396        state.pre_gate = outcomes;
8397        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8398            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8399                Ok(head) => {
8400                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8401                    state.pre_gate_commit = Some(head);
8402                }
8403                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8404            },
8405            Ok(false) => {}
8406            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8407        }
8408        if let Err(e) = state.save() {
8409            tracing::warn!("could not persist the pre_gate record: {e:#}");
8410        }
8411    }
8412
8413    /// Run `verify.gate` once against the winner's current tree, logging one
8414    /// event per command. Empty when nothing is configured.
8415    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8416        let wait = self.repository_wait();
8417        run_landing_gate(&mut self.state, winner, &wait).await
8418    }
8419
8420    /// One bounded fix round for a failing gate.
8421    ///
8422    /// The fixer is told the failure came from the gate itself, not from a
8423    /// reviewer, and is shown the failed commands, their exit codes and a tail
8424    /// of their output - whatever `[verify].gate` holds, nothing here knows
8425    /// what those commands run. Only a normal non-zero exit that printed
8426    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8427    /// command or a full disk says nothing about the code, and a fixer sent
8428    /// after it can only appease the machine. The round is judged by what git
8429    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8430    /// again before the gate does, so a fix cannot trade a green gate for a
8431    /// red e2e unnoticed.
8432    async fn gate_fix_round(
8433        &mut self,
8434        winner: &Candidate,
8435        outcomes: &[CommandOutcome],
8436    ) -> Result<GateFix> {
8437        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8438            g.phase == crate::repair_budget::Phase::Fixing
8439                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8440        }) {
8441            self.state.gate_fixes.len().saturating_add(1)
8442        } else {
8443            self.state.config.graph.gate_fix_rounds
8444        };
8445        let spent = self.state.gate_fixes.len();
8446        if spent >= cap {
8447            if cap > 0 {
8448                self.state.event(
8449                    "gate",
8450                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8451                );
8452            }
8453            return Ok(GateFix::Stop);
8454        }
8455        if !gate_fixable(outcomes) {
8456            self.state.event(
8457                "gate",
8458                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8459                 command or similar); not spending a fix round on it",
8460            );
8461            return Ok(GateFix::Stop);
8462        }
8463        let min_free = self.state.config.disk.min_free_bytes;
8464        if min_free > 0 {
8465            match crate::disk::free_bytes(&winner.worktree) {
8466                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8467                Ok(free) => {
8468                    self.state.event(
8469                        "gate",
8470                        format!(
8471                            "only {free} bytes free ({min_free} required by `[disk] \
8472                             min_free_bytes`); not spending a fix round on a failure the disk \
8473                             may explain"
8474                        ),
8475                    );
8476                    return Ok(GateFix::Stop);
8477                }
8478                Err(e) => {
8479                    self.state.event(
8480                        "gate",
8481                        format!("free disk space could not be measured ({e:#}); no fix round"),
8482                    );
8483                    return Ok(GateFix::Stop);
8484                }
8485            }
8486        }
8487
8488        let attempt = spent + 1;
8489        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8490        let base = self.landing_base();
8491        let fix_prompt = prompt::gate_fix_in(
8492            &self.state.instruction,
8493            &failed,
8494            attempt,
8495            cap,
8496            &self.state.config.graph.language,
8497            &self.state.config.graph.github_language,
8498        );
8499        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8500        let sessions = self.state.config.graph.sessions;
8501        let artifacts = self.merge_repair_artifacts();
8502        self.state.event(
8503            "gate",
8504            format!("gate failed; gate-fix round {attempt} of {cap}"),
8505        );
8506        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8507        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8508        // Persist before invoking the fixer: it may commit and then be
8509        // interrupted before its result (or the following e2e) is saved.
8510        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8511            head: before.clone(),
8512            passed: false,
8513            outcomes: Vec::new(),
8514        });
8515        self.state.save()?;
8516        let (_, seat, out) = self
8517            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8518                prompt: fix_prompt.clone(),
8519                spec,
8520                seat,
8521                cwd: winner.worktree.clone(),
8522                timeout,
8523                allow_write: true,
8524                sessions,
8525                artifacts: artifacts.clone(),
8526                stem: format!("gate-fix-{attempt}"),
8527                handover: None,
8528            })
8529            .await;
8530        // The fixer was never asked: no round is spent (nothing is recorded)
8531        // and a later reentry asks the same agent again.
8532        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8533            return Ok(GateFix::Defer);
8534        }
8535        let mut record = GateFixRecord {
8536            agent: seat.agent.clone(),
8537            failed,
8538            notes: String::new(),
8539            committed: false,
8540            error: None,
8541        };
8542        match out {
8543            AgentOutcome::Ok(o) => {
8544                // A missing report is not a failed fix: the round is judged
8545                // by the tree below, and the report only carries prose.
8546                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8547                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8548                }
8549            }
8550            AgentOutcome::Dropped(_) => {
8551                record.error = Some("the CLI dropped the stream".to_owned());
8552            }
8553            AgentOutcome::Quota(o) => {
8554                self.state.quota.push(QuotaLoss::from_report(
8555                    seat.key.clone(),
8556                    "gate-fix",
8557                    o.quota.as_ref(),
8558                ));
8559                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8560            }
8561            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8562        }
8563        self.state.seats.insert(seat.key.clone(), seat);
8564        let coordination = self.coordinate(&winner.worktree).await?;
8565        if let Ok(r) = git::rescue_commit(
8566            &winner.worktree,
8567            &format!("magi: gate fix {attempt} (uncommitted work)"),
8568        )
8569        .await
8570        {
8571            self.state.note_withheld("gate-fix", &r.withheld);
8572        }
8573        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8574        record.committed = after != before;
8575        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8576        let note = record.error.clone();
8577        self.state.gate_fixes.push(record);
8578        self.state.save()?;
8579        if !changed {
8580            self.state.event(
8581                "gate",
8582                match note {
8583                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8584                    None => format!("gate-fix round {attempt}: the tree did not change"),
8585                },
8586            );
8587            return Ok(GateFix::Stop);
8588        }
8589        self.state.event(
8590            "gate",
8591            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8592        );
8593
8594        drop(coordination);
8595        let wait = self.repository_wait();
8596        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8597    }
8598
8599    // ---------------------------------------------------------------- merge
8600
8601    /// One read-only rewrite by the summary's author; a text the gate still
8602    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8603    /// applies on their word or on silence. `None` means the run is parked on
8604    /// that question and nothing may be pushed or opened yet.
8605    async fn guarded_pr_message(
8606        &mut self,
8607        winner: &Candidate,
8608        facts: Option<&BranchFacts>,
8609        posting: bool,
8610    ) -> Result<Option<PrMessage>> {
8611        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8612        let head = match final_head.as_deref() {
8613            Some(h) => crate::pr_outcome::FinalHead::At(h),
8614            None => crate::pr_outcome::FinalHead::Unknown,
8615        };
8616        // Only a pull request that is about to be posted pays for a summary.
8617        let summarized = if posting {
8618            Some(self.summarize_change(winner, final_head.as_deref()).await)
8619        } else {
8620            None
8621        };
8622        // A busy shared resource is not a permanent cause: publishing an
8623        // "unavailable" description would stick (a later head move never
8624        // edits the body). Stop before anything is pushed or opened; the
8625        // run is `Blocked` and a resume asks the summary again.
8626        if self.state.resource_block.is_some() {
8627            self.state.event(
8628                "pr-summary",
8629                "a shared resource was busy; nothing was published, resume to summarize again",
8630            );
8631            return Ok(None);
8632        }
8633        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8634            anyhow::bail!(
8635                "the final commit moved during public description generation; retry before publishing"
8636            );
8637        }
8638        let note = match (&summarized, &self.state.change_summary) {
8639            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8640            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8641            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8642            (None, _) => ChangeNote::NotAsked,
8643        };
8644        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8645        if !posting {
8646            let identity = crate::scrub::Identity::current();
8647            return Ok(Some(PrMessage {
8648                title: crate::scrub::scrub(&pr.title, &identity),
8649                body: crate::scrub::scrub(&pr.body, &identity),
8650            }));
8651        }
8652        // This very text was already rejected and asked about: no second
8653        // judge call, rewrite or question, just carry the record on.
8654        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8655        if let Some(known) = self
8656            .state
8657            .github_text
8658            .clone()
8659            .filter(|g| g.fingerprint == raw_fp)
8660        {
8661            return self.settle_withheld(winner, &pr, known).await;
8662        }
8663        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8664        let mut violations =
8665            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8666        let identity = crate::scrub::Identity::current();
8667        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8668            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8669            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8670        {
8671            violations.push(crate::github_text::Violation::SensitiveData);
8672        }
8673        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8674        // only a language violation starts one; sensitive data alone goes
8675        // straight to the owner, with the redacted text as the fallback.
8676        let has_language = violations
8677            .iter()
8678            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8679        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8680            self.state.event(
8681                "github-text",
8682                if has_language {
8683                    format!("description rejected: {violations:?}; requesting one rewrite")
8684                } else {
8685                    format!("description rejected: {violations:?}; asking the owner")
8686                },
8687            );
8688            let mut rewritten = false;
8689            let mut rejected_rewrite = None;
8690            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8691            {
8692                let output = text;
8693                if let Some(title) = summary_title(&output) {
8694                    // Judge the redacted candidate that will actually be posted.
8695                    // Sensitive data alone must not discard a useful rewrite.
8696                    let raw_body = summary_without_title(&output);
8697                    let clean_title = crate::scrub::scrub(&title, &identity);
8698                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8699                    let redacted = clean_title != title || body != raw_body;
8700                    let title = clean_title;
8701                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8702                    if !complete {
8703                        self.state.event(
8704                            "github-text",
8705                            "description rewrite omitted required findings",
8706                        );
8707                    }
8708                    if !body.trim().is_empty() {
8709                        body.push_str("\n\n");
8710                        body.push_str(&public_pr_evidence(&self.state, head));
8711                    }
8712                    let marker = format!("magi:run/{}", self.state.id);
8713                    if !body.trim().is_empty() && !body.contains(&marker) {
8714                        body.push_str(&format!("\n\n{marker}"));
8715                    }
8716                    let rewrite = PrMessage {
8717                        title: title.clone(),
8718                        body: body.clone(),
8719                    };
8720                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8721                    let mut failed = crate::github_text::check_for(
8722                        &self.state.config.graph,
8723                        &title,
8724                        &body,
8725                        redecision,
8726                    );
8727                    if (!crate::github_text::shareable(&title)
8728                        || !crate::github_text::shareable(&body))
8729                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8730                    {
8731                        failed.push(crate::github_text::Violation::SensitiveData);
8732                    }
8733                    if complete
8734                        && !title.trim().is_empty()
8735                        && !body.trim().is_empty()
8736                        && !failed.is_empty()
8737                    {
8738                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8739                            title: title.clone(),
8740                            body: body.clone(),
8741                            failed: failed
8742                                .iter()
8743                                .map(|v| crate::github_text::category(*v).to_owned())
8744                                .collect(),
8745                        });
8746                    }
8747                    if complete
8748                        && !title.trim().is_empty()
8749                        && !raw_body.trim().is_empty()
8750                        && failed.is_empty()
8751                    {
8752                        decision = redecision;
8753                        pr = PrMessage { title, body };
8754                        rewritten = true;
8755                        self.state
8756                            .event("github-text", "description rewrite passed");
8757                        if redacted {
8758                            self.state.event(
8759                                "github-text",
8760                                "sensitive data removed from description rewrite",
8761                            );
8762                        }
8763                    }
8764                }
8765            }
8766            if !rewritten {
8767                self.state.event(
8768                    "github-text",
8769                    "description rewrite unavailable or rejected; asking the owner",
8770                );
8771                if let Some(w) =
8772                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8773                {
8774                    // A different text than the one asked about before: that
8775                    // question no longer describes anything.
8776                    self.retire_github_text_question("the withheld text changed");
8777                    let record = crate::run::GithubTextAsk {
8778                        rewrite: rejected_rewrite,
8779                        fingerprint: raw_fp,
8780                        title: w.title,
8781                        body: w.body,
8782                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8783                        question: None,
8784                        asks: 0,
8785                        resolved: false,
8786                        chosen_title: None,
8787                        choice: None,
8788                    };
8789                    return self.settle_withheld(winner, &pr, record).await;
8790                }
8791            }
8792        }
8793        let (title, body) =
8794            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8795        Ok(Some(PrMessage { title, body }))
8796    }
8797
8798    /// Describe the whole change from its final diff in either run mode.
8799    ///
8800    /// The summary is tied to the final commit and to the merge base of the
8801    /// resolved remote base; a record for any other pair is dropped and the
8802    /// agent asked again, so a description never outlives the diff it was
8803    /// written from. The seat is read-only and the prompt tells it to read the
8804    /// diff and the code around it. `[roles] synthesizer` selects the agents
8805    /// (unset: the default pick), each asked at most once, advancing on an
8806    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8807    /// carries the reason it is unavailable; nothing is invented then.
8808    async fn summarize_change(
8809        &mut self,
8810        winner: &Candidate,
8811        head: Option<&str>,
8812    ) -> std::result::Result<(), String> {
8813        let Some(head) = head else {
8814            return Err("the final commit could not be read".to_owned());
8815        };
8816        let remote = self.state.config.merge.remote.clone();
8817        let tracking = format!("{remote}/{}", self.state.base_branch);
8818        if !git::rev_exists(&winner.worktree, &tracking).await {
8819            return Err(format!("the remote base `{tracking}` could not be read"));
8820        }
8821        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8822            Ok(b) if !b.is_empty() => b,
8823            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8824        };
8825        let evidence = crate::pr_outcome::summary_evidence(
8826            &self.state,
8827            crate::pr_outcome::FinalHead::At(head),
8828        );
8829        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8830        if self
8831            .state
8832            .change_summary
8833            .as_ref()
8834            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8835        {
8836            return Ok(());
8837        }
8838        if self.state.change_summary.take().is_some() {
8839            self.state.event(
8840                "pr-summary",
8841                "the final diff or evidence changed; the earlier change summary was discarded",
8842            );
8843        }
8844        // Save the source snapshot even if no generation agent is available.
8845        if let Err(e) = write_artifact(
8846            &self.state,
8847            &format!("public-description-{fingerprint}.md"),
8848            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8849        ) {
8850            tracing::warn!("could not save public description evidence: {e:#}");
8851        }
8852        let chain = match agent::pick_chain(
8853            &self.state.config.agents,
8854            self.state.config.roles.synthesizer.as_ref(),
8855            &agent::installed,
8856            "synthesizer",
8857        ) {
8858            Ok(chain) => chain,
8859            Err(e) => {
8860                tracing::warn!("no agent for the change summary: {e:#}");
8861                self.state.event("pr-summary", "no summary agent available");
8862                return Err("no agent was available to summarize it".to_owned());
8863            }
8864        };
8865        let subjects = git::subjects(&winner.worktree, &base, head)
8866            .await
8867            .unwrap_or_default();
8868        let stat = git::diff_stat(&winner.worktree, &base, head)
8869            .await
8870            .unwrap_or_default();
8871        let stat: String = stat.chars().take(6_000).collect();
8872        let mut prompt = crate::prompt::change_summary_in(
8873            &base,
8874            head,
8875            &subjects,
8876            stat.trim(),
8877            &self.state.config.graph.github_language,
8878        );
8879        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8880        let sessions = self.state.config.graph.sessions;
8881        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8882        let prompts = self.state.config.prompts.clone();
8883        let cache = self.state.config.cache_dir();
8884        let run = self.state.id.clone();
8885        for (i, spec) in chain.into_iter().enumerate() {
8886            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8887            let github_spec = match agent::for_github_text(
8888                &spec,
8889                &self.state.config.graph.github_language,
8890                &winner.worktree,
8891            ) {
8892                Ok(spec) => spec,
8893                Err(e) => {
8894                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8895                    continue;
8896                }
8897            };
8898            let job = SeatJob {
8899                spec: github_spec,
8900                seat,
8901                cwd: winner.worktree.clone(),
8902                prompt: prompt.clone(),
8903                timeout,
8904                allow_write: false,
8905                sessions,
8906                artifacts: agent::artifacts_dir(&self.state.dir()),
8907                stem: if i == 0 {
8908                    "change-summary".to_owned()
8909                } else {
8910                    format!("change-summary-{}", spec.id)
8911                },
8912                handover: None,
8913            };
8914            let ctx = WaveCtx {
8915                run: &run,
8916                node: "pr-summary",
8917                prompts: &prompts,
8918                cache: cache.as_deref(),
8919                round: None,
8920                carry_seats: false,
8921                pauses: Some((&self.pause, &self.interrupt)),
8922            };
8923            let (seat, outcome) =
8924                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8925            self.state.seats.insert(seat.key.clone(), seat);
8926            let text = match outcome {
8927                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8928                AgentOutcome::ResourceBlocked(why) => {
8929                    // No agent was asked: the run stops resumably before
8930                    // anything is published (see `guarded_pr_message`).
8931                    if self.state.resource_block.is_none() {
8932                        self.state.resource_block = Some(crate::run::ResourceBlock {
8933                            cause: crate::run::ResourceBlockCause::Other,
8934                            node: "pr-summary".to_owned(),
8935                            kind: "shared resource".to_owned(),
8936                            detail: why,
8937                            at: Timestamp::now(),
8938                        });
8939                    }
8940                    return Err("a shared resource was busy".to_owned());
8941                }
8942                _ => continue,
8943            };
8944            let blind = &self.state.config.blind;
8945            let Some(title) = summary_title(&text) else {
8946                continue;
8947            };
8948            let body: String = summary_without_title(&text)
8949                .lines()
8950                .filter(|l| !l.contains("magi:run/"))
8951                .collect::<Vec<_>>()
8952                .join("\n");
8953            let body = blind::sanitize_prose(body.trim(), blind);
8954            let title = blind::sanitize_prose(&title, blind);
8955            if body.trim().is_empty() || title.trim().is_empty() {
8956                continue;
8957            }
8958            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8959                continue;
8960            }
8961            // The branch must still be the one the agent was shown.
8962            if git::rev_parse(&winner.worktree, "HEAD")
8963                .await
8964                .ok()
8965                .as_deref()
8966                != Some(head)
8967            {
8968                return Err("the branch moved while the summary was written".to_owned());
8969            }
8970            if git::merge_base(&winner.worktree, &tracking, head)
8971                .await
8972                .ok()
8973                .as_deref()
8974                != Some(base.as_str())
8975                || crate::pr_outcome::summary_evidence(
8976                    &self.state,
8977                    crate::pr_outcome::FinalHead::At(head),
8978                ) != evidence
8979            {
8980                return Err("the summary evidence changed while it was written".to_owned());
8981            }
8982            self.state.change_summary = Some(crate::run::ChangeSummary {
8983                head: head.to_owned(),
8984                base,
8985                evidence: fingerprint,
8986                title,
8987                body,
8988            });
8989            self.state.event(
8990                "pr-summary",
8991                format!("change summary written by {} for the final diff", spec.id),
8992            );
8993            if let Err(e) = self.state.save() {
8994                tracing::warn!("could not save the change summary: {e:#}");
8995            }
8996            return Ok(());
8997        }
8998        self.state
8999            .event("pr-summary", "no agent produced a change summary");
9000        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9001    }
9002
9003    /// One read-only rewrite request for a withheld description. The
9004    /// implementer's own conversation is tried first when it exists and can be
9005    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9006    /// writes it in a separate conversation, skipping the implementer's agent.
9007    /// The prompt carries the whole text, so no agent needs any memory. Each
9008    /// agent is asked at most once and the chain advances on an error, quota,
9009    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9010    /// that has run out of quota must not end the rewrite. Nothing is recorded
9011    /// as a handover.
9012    async fn rewrite_pr_text(
9013        &mut self,
9014        winner: &Candidate,
9015        pr: &PrMessage,
9016        violations: &[crate::github_text::Violation],
9017    ) -> Option<String> {
9018        let prompt = format!(
9019            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9020            pr.title, pr.body
9021        );
9022        let sessions = self.state.config.graph.sessions;
9023        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9024        // (spec, the resumed implementer's seat). A fallback seat is created
9025        // only when its turn comes, so an unused fallback leaves no record.
9026        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9027        let key = format!("impl-{}", winner.label);
9028        if let Some(spec) = self
9029            .state
9030            .config
9031            .agents
9032            .iter()
9033            .find(|a| a.id == winner.agent)
9034            .cloned()
9035            && let Some(existing) = self.state.seats.get(&key)
9036            && existing.agent == spec.id
9037            && agent::has_session(spec.kind, existing, sessions)
9038        {
9039            attempts.push((spec, Some(existing.clone())));
9040        }
9041        match agent::pick_chain(
9042            &self.state.config.agents,
9043            self.state.config.roles.language_judge.as_ref(),
9044            &agent::installed,
9045            "language_judge",
9046        ) {
9047            Ok(chain) => {
9048                for spec in chain {
9049                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9050                        continue;
9051                    }
9052                    attempts.push((spec, None));
9053                }
9054            }
9055            Err(e) if attempts.is_empty() => {
9056                tracing::warn!("no agent for the description rewrite: {e:#}");
9057                self.state
9058                    .event("github-text", "no rewrite agent available");
9059                return None;
9060            }
9061            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9062        }
9063        let prompts = self.state.config.prompts.clone();
9064        let cache = self.state.config.cache_dir();
9065        let run = self.state.id.clone();
9066        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9067            let resumed = resumed_seat.is_some();
9068            let seat = resumed_seat.unwrap_or_else(|| {
9069                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9070            });
9071            let github_spec = match agent::for_github_text(
9072                &spec,
9073                &self.state.config.graph.github_language,
9074                &winner.worktree,
9075            ) {
9076                Ok(spec) => spec,
9077                Err(e) => {
9078                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9079                    continue;
9080                }
9081            };
9082            let job = SeatJob {
9083                spec: github_spec,
9084                seat,
9085                cwd: winner.worktree.clone(),
9086                prompt: prompt.clone(),
9087                timeout: retry_budget(timeout_full, resumed),
9088                allow_write: false,
9089                sessions,
9090                artifacts: agent::artifacts_dir(&self.state.dir()),
9091                stem: if i == 0 {
9092                    "github-text-rewrite".to_owned()
9093                } else {
9094                    format!("github-text-rewrite-{}", spec.id)
9095                },
9096                handover: None,
9097            };
9098            let ctx = WaveCtx {
9099                run: &run,
9100                node: "github-text",
9101                prompts: &prompts,
9102                cache: cache.as_deref(),
9103                round: None,
9104                carry_seats: false,
9105                pauses: Some((&self.pause, &self.interrupt)),
9106            };
9107            let (seat, outcome) =
9108                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9109            self.state.seats.insert(seat.key.clone(), seat);
9110            match outcome {
9111                AgentOutcome::Ok(o)
9112                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9113                {
9114                    return Some(o.text);
9115                }
9116                AgentOutcome::ResourceBlocked(_) => return None,
9117                _ => {}
9118            }
9119        }
9120        None
9121    }
9122
9123    /// Abandon the standing posting-gate question, if any.
9124    fn retire_github_text_question(&mut self, why: &str) {
9125        let Some(id) = self
9126            .state
9127            .github_text
9128            .as_ref()
9129            .and_then(|g| g.question.clone())
9130        else {
9131            return;
9132        };
9133        let _ = ask::Questions::open().update(&id, |q| {
9134            q.abandon(why);
9135            Ok(())
9136        });
9137    }
9138
9139    /// Carry a withheld text to its end: file the question, park while it is
9140    /// open, and on an answer, silence or timeout pick the text to post. The
9141    /// decision is saved before it is used, so it applies once.
9142    async fn settle_withheld(
9143        &mut self,
9144        winner: &Candidate,
9145        pr: &PrMessage,
9146        mut g: crate::run::GithubTextAsk,
9147    ) -> Result<Option<PrMessage>> {
9148        use crate::github_text as gt;
9149        let store = ask::Questions::open();
9150        let timeout = self.state.config.graph.answer_timeout;
9151        if !g.resolved {
9152            let mut standing = g
9153                .question
9154                .as_ref()
9155                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9156            if let Some(q) = standing.as_ref()
9157                && q.status.open()
9158                && gt::expired(q, timeout)
9159            {
9160                let why = format!("no answer within {}s of asking", timeout.max(1));
9161                standing = store
9162                    .update(&q.id, |q| {
9163                        q.abandon(&why);
9164                        Ok(())
9165                    })
9166                    .ok()
9167                    .map(|(q, ())| q);
9168            }
9169            let mut ask_again: Option<Option<Vec<String>>> = None;
9170            match standing {
9171                // Recorded but never written (a stop between the two saves),
9172                // or removed: ask again without spending the retry.
9173                None if g.asks <= 1 => {
9174                    g.asks = 0;
9175                    ask_again = Some(None);
9176                }
9177                None => {
9178                    g.resolved = true;
9179                    g.chosen_title = None;
9180                    g.choice = Some(GithubTextChoice::Fallback);
9181                }
9182                Some(q) if q.status.open() => {
9183                    self.park_on_github_text(&g)?;
9184                    return Ok(None);
9185                }
9186                Some(q) => match gt::read_reply(&q) {
9187                    gt::Reply::Fallback => {
9188                        g.resolved = true;
9189                        g.chosen_title = None;
9190                        g.choice = Some(GithubTextChoice::Fallback);
9191                    }
9192                    gt::Reply::Original => {
9193                        g.resolved = true;
9194                        g.chosen_title = None;
9195                        g.choice = Some(GithubTextChoice::Original);
9196                    }
9197                    gt::Reply::Rewrite => {
9198                        g.resolved = true;
9199                        g.chosen_title = None;
9200                        g.choice = Some(if g.rewrite.is_some() {
9201                            GithubTextChoice::Rewrite
9202                        } else {
9203                            GithubTextChoice::Fallback
9204                        });
9205                    }
9206                    gt::Reply::Title(text) => {
9207                        let first = text
9208                            .lines()
9209                            .map(str::trim)
9210                            .find(|l| !l.is_empty())
9211                            .unwrap_or("");
9212                        let decision = if first.is_empty() {
9213                            None
9214                        } else {
9215                            let candidate = PrMessage {
9216                                title: first.to_owned(),
9217                                body: String::new(),
9218                            };
9219                            self.judge_pr_language(&winner.worktree, &candidate).await
9220                        };
9221                        match gt::vet_title(
9222                            &text,
9223                            decision,
9224                            self.state.config.graph.github_english(),
9225                        ) {
9226                            Ok(title) => {
9227                                g.resolved = true;
9228                                g.chosen_title = Some(title);
9229                                g.choice = Some(GithubTextChoice::MyTitle);
9230                            }
9231                            Err(cats) if g.asks < 2 => {
9232                                ask_again = Some(Some(
9233                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9234                                ))
9235                            }
9236                            Err(_) => {
9237                                g.resolved = true;
9238                                g.chosen_title = None;
9239                                g.choice = Some(GithubTextChoice::Fallback);
9240                            }
9241                        }
9242                    }
9243                },
9244            }
9245            if let Some(rejected) = ask_again {
9246                // A retry (after a rejected replacement title) is still about
9247                // the original text: the same ORIGINAL / FALLBACK comparison.
9248                let w = gt::Withheld {
9249                    title: g.title,
9250                    body: g.body,
9251                    categories: g.categories.clone(),
9252                };
9253                // Named by the original text's fingerprint alone, so a resume
9254                // rewrites the same file and the detail stays identical.
9255                let artifact = match crate::run::write_artifact(
9256                    &self.state,
9257                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9258                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9259                ) {
9260                    Ok(p) => Some(p),
9261                    Err(e) => {
9262                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9263                        None
9264                    }
9265                };
9266                let (detail, comparison) = gt::question_content(
9267                    &w,
9268                    &pr.title,
9269                    &pr.body,
9270                    &self.state.id,
9271                    rejected.as_deref(),
9272                    artifact.as_deref(),
9273                    g.rewrite.as_ref(),
9274                );
9275                let mut q = ask::Question::new(
9276                    self.state.id.clone(),
9277                    gt::ASK_NODE.to_owned(),
9278                    gt::ASK_SEAT.to_owned(),
9279                    gt::question_summary(&self.state.config.graph.language, &w),
9280                    detail,
9281                    gt::question_choices(&w, g.rewrite.as_ref()),
9282                );
9283                q.posting_comparison = Some(comparison);
9284                q.answer_timeout = timeout;
9285                // Recorded before it is written: a stop in between leaves a
9286                // record naming a question the resume then files again.
9287                g.question = Some(q.id.clone());
9288                g.asks += 1;
9289                self.state.github_text = Some(g.clone());
9290                self.state.event(
9291                    "github-text",
9292                    format!("asking the owner about the withheld text ({})", q.short()),
9293                );
9294                self.state.save()?;
9295                store
9296                    .put_executable(&mut q)
9297                    .context("file the posting-gate question")?;
9298                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9299                    tracing::warn!(
9300                        "could not notify about posting-gate question {}: {e:#}",
9301                        q.short()
9302                    );
9303                }
9304                self.park_on_github_text(&g)?;
9305                return Ok(None);
9306            }
9307        }
9308        let resolved = g.chosen_title.clone();
9309        let original = g.choice == Some(GithubTextChoice::Original);
9310        // Field names only: the event log never records the text.
9311        self.state.event(
9312            "github-text",
9313            if original {
9314                format!(
9315                    "posting the original text for: {}",
9316                    match (g.title, g.body) {
9317                        (true, true) => "title, description",
9318                        (true, false) => "title",
9319                        _ => "description",
9320                    }
9321                )
9322            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9323                "posting the saved rewrite title and description".to_owned()
9324            } else if resolved.is_some() {
9325                "posting the owner's replacement title".to_owned()
9326            } else {
9327                "posting the neutral text for the withheld fields".to_owned()
9328            },
9329        );
9330        self.state.github_text = Some(g.clone());
9331        self.state.save()?;
9332        if g.choice == Some(GithubTextChoice::Rewrite)
9333            && let Some(r) = g.rewrite.as_ref()
9334        {
9335            return Ok(Some(PrMessage {
9336                title: r.title.clone(),
9337                body: r.body.clone(),
9338            }));
9339        }
9340        let w = gt::Withheld {
9341            title: g.title,
9342            body: g.body,
9343            categories: g.categories.clone(),
9344        };
9345        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9346        if let Some(t) = resolved {
9347            title = t;
9348        }
9349        // Every withheld field is now neutral or vetted English.
9350        let english = gt::LanguageDecision {
9351            title_english: true,
9352            body_english: true,
9353            body_complete: true,
9354        };
9355        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9356        if original {
9357            // The owner's explicit choice overrides the gate for the fields it
9358            // was made for; the others keep what the gate produced.
9359            if g.title {
9360                title = pr.title.clone();
9361            }
9362            if g.body {
9363                body = pr.body.clone();
9364            }
9365        }
9366        Ok(Some(PrMessage { title, body }))
9367    }
9368
9369    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9370        self.state.github_text = Some(g.clone());
9371        self.state.end_pass_parked(ParkCause::Question);
9372        self.state.parked = true;
9373        self.state.event(
9374            "github-text",
9375            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9376        );
9377        self.state.save()
9378    }
9379
9380    /// Ask `[roles] language_judge` about this text and record which source
9381    /// decided; the text itself is never recorded.
9382    async fn judge_pr_language(
9383        &mut self,
9384        cwd: &Path,
9385        pr: &PrMessage,
9386    ) -> Option<crate::github_text::LanguageDecision> {
9387        self.state.config.roles.language_judge.as_ref()?;
9388        if !self.state.config.graph.language_gate() {
9389            return None;
9390        }
9391        let decision =
9392            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9393        self.state.event(
9394            "github-text",
9395            if decision.is_some() {
9396                "language decided by the language judge"
9397            } else {
9398                "language judge unavailable; using built-in heuristics"
9399            },
9400        );
9401        decision
9402    }
9403
9404    async fn merge(&mut self) -> Result<()> {
9405        if self.resource_halt()? || self.hold_base_sync().await? {
9406            return Ok(());
9407        }
9408        // Same reasoning as `gate`: ask the review and gate records directly
9409        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9410        // skip can rewrite on reentry to something that no longer says
9411        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9412        // so a hand-off (open findings, green verification) reaches merge
9413        // exactly like a genuinely clean round does.
9414        //
9415        // A run resumed mid-`land` never reaches here at all: `execute`
9416        // recognises `RunStatus::Landing` before it even calls `prep`, and
9417        // routes straight to `run_land` instead. That has to happen a level
9418        // up from this function, not with a check in here, because
9419        // `review_loop`'s own status recomputation (see its doc) runs
9420        // *before* `merge` on every reentry and would otherwise overwrite
9421        // the `Landing` marker with `Gating` before this node ever saw it.
9422        if self
9423            .state
9424            .base_sync
9425            .as_ref()
9426            .is_some_and(|s| s.conflict.is_some())
9427            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9428                != Some(RunStatus::Gating)
9429            // `gate_ran == false` is not "passed" - `gate` leaves it false
9430            // both before it has ever run and when its last attempt was
9431            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9432            // permission to merge on nothing but the review record. Only a
9433            // gate that actually ran may proceed; an intentional local
9434            // review can also finish with verification explicitly not configured.
9435            // `RunState::gate_status` computes that distinction.
9436            || !self.state.gate_allows_progress()
9437        {
9438            return Ok(());
9439        }
9440        // This node's own record, not `status`: `status == Ready` is not
9441        // unique to the harmless `MergeMode::None` path this line was
9442        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9443        // run's PR was closed without merging — and on that run `mode` is
9444        // still `Pr`, so a reentry that fell through here would push and
9445        // open a second pull request. `self.state.merge` is set exactly once
9446        // this node (or `land`) has already produced a verdict, under every
9447        // mode, which is what "already done" actually means here.
9448        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9449            return Ok(());
9450        }
9451        let Some(winner) = self.state.winner().cloned() else {
9452            return Ok(());
9453        };
9454        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9455        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9456            self.state.status = RunStatus::Blocked;
9457            self.state.event(
9458                "gate",
9459                "winner changed after gate; fresh verification required",
9460            );
9461            self.state.save()?;
9462            return Ok(());
9463        }
9464        let wait = self.repository_wait();
9465        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9466            return Ok(());
9467        }
9468        let repo = self.state.repo.clone();
9469        let base = self.state.base_branch.clone();
9470        let mode = self.state.config.merge.mode;
9471        let style = self.state.config.merge.style;
9472        let facts = if is_review_run(&self.state) {
9473            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9474            let start = review_base(
9475                &repo,
9476                &self.state.config.merge.remote,
9477                &base,
9478                &self.state.base_commit,
9479                &winner.branch,
9480            )
9481            .await;
9482            branch_facts(&repo, &start, &winner.branch).await
9483        } else {
9484            None
9485        };
9486        // `None` when the base could not be freshly read: then an adopted
9487        // pull request's title is left alone.
9488        let leaked = if is_review_run(&self.state) {
9489            leaked_subjects(&self.state, &winner.branch).await
9490        } else {
9491            Some(Vec::new())
9492        };
9493        // `None`: parked on the owner's word about a withheld title or body.
9494        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9495        // the resume arrives here again.
9496        let Some(pr) = self
9497            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9498            .await?
9499        else {
9500            return Ok(());
9501        };
9502        let message = pr.commit_message();
9503
9504        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9505        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9506            return Ok(());
9507        }
9508        let outcome = match mode {
9509            MergeMode::None => MergeOutcome {
9510                mode,
9511                ok: true,
9512                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9513                empty: false,
9514            },
9515            MergeMode::Pr | MergeMode::Local
9516                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9517            {
9518                MergeOutcome {
9519                    mode,
9520                    ok: false,
9521                    detail: empty_candidate_detail(&self.state, &base),
9522                    empty: true,
9523                }
9524            }
9525            MergeMode::Local => {
9526                let on = git::current_branch(&repo).await?;
9527                if on.as_deref() != Some(base.as_str()) {
9528                    MergeOutcome {
9529                        mode,
9530                        ok: false,
9531                        detail: format!(
9532                            "{} has {} checked out, not the base branch {base}",
9533                            repo.display(),
9534                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9535                        ),
9536                        empty: false,
9537                    }
9538                } else if !git::is_clean(&repo).await? {
9539                    MergeOutcome {
9540                        mode,
9541                        ok: false,
9542                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9543                        empty: false,
9544                    }
9545                } else {
9546                    let out = match style {
9547                        MergeStyle::Merge => {
9548                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9549                        }
9550                        MergeStyle::Squash => {
9551                            git::merge_squash(&repo, &winner.branch, &message).await?
9552                        }
9553                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9554                    };
9555                    MergeOutcome {
9556                        mode,
9557                        ok: out.ok(),
9558                        detail: if out.ok() { out.stdout } else { out.stderr },
9559                        empty: false,
9560                    }
9561                }
9562            }
9563            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9564                self.state.pr_publication_pending = true;
9565                self.state.save()?;
9566                let out =
9567                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9568                self.state.pr_publication_pending = out.is_err();
9569                MergeOutcome {
9570                    mode,
9571                    ok: out.is_ok(),
9572                    detail: match out {
9573                        Ok(url) => url,
9574                        Err(e) => format!("{e:#}"),
9575                    },
9576                    empty: false,
9577                }
9578            }
9579            MergeMode::Pr => {
9580                let remote = self.state.config.merge.remote.clone();
9581                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9582                let pushed = if self.state.foreign_checkout.is_some() {
9583                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9584                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9585                } else {
9586                    git::push(&winner.worktree, &remote, &winner.branch).await?
9587                };
9588                if !pushed.ok() {
9589                    MergeOutcome {
9590                        mode,
9591                        ok: false,
9592                        detail: pushed.stderr,
9593                        empty: false,
9594                    }
9595                } else {
9596                    if self.state.foreign_checkout.is_some() {
9597                        self.state.pr_pushed_head = Some(delivery_head.clone());
9598                    }
9599                    // A retry or resume of a run whose branch already has an
9600                    // open pull request adopts it rather than failing on a
9601                    // duplicate. Only this winner branch into this base:
9602                    // `branch_for` derives the name from the run id, so a
9603                    // different run's pull request never matches.
9604                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9605                    let out = match pr_merge_plan(found) {
9606                        PrPlan::Create => {
9607                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9608                                .await
9609                        }
9610                        PrPlan::Adopt { url, title } => {
9611                            self.state
9612                                .event("merge", format!("Pr: adopted open pull request {url}"));
9613                            if self.state.github_text.as_ref().is_some_and(|g| {
9614                                g.resolved
9615                                    && g.choice == Some(GithubTextChoice::Rewrite)
9616                                    && g.rewrite.is_some()
9617                            }) {
9618                                // The owner chose the entire pair, even if the
9619                                // existing title already matches or was edited.
9620                                land::set_pr_text_verbatim(
9621                                    &winner.worktree,
9622                                    &url,
9623                                    &pr.title,
9624                                    &pr.body,
9625                                )
9626                                .await
9627                                .map(|()| url)
9628                            } else {
9629                                if title != pr.title
9630                                    && (!is_review_run(&self.state)
9631                                        || leaked
9632                                            .as_deref()
9633                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9634                                    && let Err(e) =
9635                                        if self.state.github_text.as_ref().is_some_and(|g| {
9636                                            g.resolved
9637                                                && g.title
9638                                                && g.choice == Some(GithubTextChoice::Original)
9639                                        }) {
9640                                            // The owner chose to post this title unmodified.
9641                                            land::set_pr_title_verbatim(
9642                                                &winner.worktree,
9643                                                &url,
9644                                                &pr.title,
9645                                            )
9646                                            .await
9647                                        } else {
9648                                            land::set_pr_title(
9649                                                &mut self.state,
9650                                                &winner.worktree,
9651                                                &url,
9652                                                &pr.title,
9653                                            )
9654                                            .await
9655                                        }
9656                                {
9657                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9658                                    self.state
9659                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9660                                }
9661                                Ok(url)
9662                            }
9663                        }
9664                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9665                    };
9666                    match out {
9667                        Ok(url) => MergeOutcome {
9668                            mode,
9669                            ok: true,
9670                            detail: url,
9671                            empty: false,
9672                        },
9673                        Err(e) => MergeOutcome {
9674                            mode,
9675                            ok: false,
9676                            detail: e.to_string(),
9677                            empty: false,
9678                        },
9679                    }
9680                }
9681            }
9682        };
9683
9684        self.state.status = match (mode, outcome.ok) {
9685            (MergeMode::None, _) => RunStatus::Ready,
9686            (_, true) if outcome.empty => RunStatus::Ready,
9687            (MergeMode::Pr, true) => RunStatus::Ready,
9688            (_, true) => RunStatus::Merged,
9689            (_, false) => RunStatus::Blocked,
9690        };
9691        self.state.event(
9692            "merge",
9693            format!(
9694                "{:?}: {}",
9695                mode,
9696                outcome.detail.lines().next().unwrap_or("")
9697            ),
9698        );
9699        self.state.merge = Some(outcome);
9700        self.state.save()?;
9701
9702        // The PR is open and the run would historically stop here, leaving the
9703        // operator to watch checks, feed review comments back to a fixer, and
9704        // merge. That was done by hand six times in one session before this
9705        // existed. Opt-in, because merging is the one irreversible thing magi
9706        // can do to a repository.
9707        if self.state.config.graph.land
9708            && mode == MergeMode::Pr
9709            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9710        {
9711            self.run_land().await?;
9712        }
9713        // `run_land` may have left `status` at `Landing` - still waiting on
9714        // CI or the owner's approval, not actually settled - so this has to
9715        // read whatever `status` ended up as here, not the `Merged` this
9716        // function set a few lines up.
9717        self.settle_questions();
9718        Ok(())
9719    }
9720
9721    /// Enter `land`.
9722    ///
9723    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9724    /// a resumed run's re-entry. `land::land` itself is what serialises the
9725    /// two git-mutating moments inside the loop — the rebase push and
9726    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9727    /// to hold a lock across the whole call, and doing so would serialise
9728    /// this run's CI wait against a *different* run's land-approval resume
9729    /// in the same repository, which is exactly the "must not wait on
9730    /// another task" property the daemon's slot-freeing exists to give.
9731    async fn run_land(&mut self) -> Result<()> {
9732        crate::pr_head::ensure_owner(&self.state)?;
9733        let url = self
9734            .state
9735            .pr
9736            .as_ref()
9737            .map(|pr| pr.url.clone())
9738            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9739            .unwrap_or_default();
9740        let url = url.lines().next().unwrap_or("").trim().to_owned();
9741        if !url.starts_with("http") {
9742            return Ok(());
9743        }
9744        // A land failure is not a lost run: the work is on a branch and the
9745        // pull request is open, which is exactly where a human takes over.
9746        loop {
9747            match land::land(&mut self.state, &url).await {
9748                Ok(_)
9749                    if self
9750                        .state
9751                        .merge_repairs
9752                        .last()
9753                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9754                {
9755                    if self.repair_merge_review().await? {
9756                        continue;
9757                    }
9758                }
9759                Ok(_) if self.state.resource_block.is_some() => {
9760                    return self.stop_for_resource();
9761                }
9762                Ok(pr) if self.state.parked => {
9763                    // `land` already saved the parked marker; nothing here
9764                    // overrides `status` back to a terminal value while an
9765                    // approval is still outstanding.
9766                    let _ = pr;
9767                }
9768                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9769                    self.state.status = RunStatus::Superseded;
9770                    self.state.save()?;
9771                }
9772                Ok(pr) => {
9773                    self.state.status = match pr.state {
9774                        land::PrLifecycle::Merged => RunStatus::Merged,
9775                        _ => RunStatus::Blocked,
9776                    };
9777                    // Downstream of a confirmed merge only - see
9778                    // `bump::should_release_bump`'s own doc for why this one
9779                    // check covers all three of `land`'s success paths.
9780                    // Best-effort: the run already landed, so a failure here
9781                    // (the decision call, `gh`, `cargo`) is recorded and never
9782                    // turns a landed run into a failed one.
9783                    if bump::should_release_bump(self.state.status)
9784                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9785                    {
9786                        // The event is the run's own record. Not-eligible cases
9787                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9788                        // `Err` is a bump that was tried and failed:
9789                        // `after_merge` itself raises the operator notice for
9790                        // that, whether or not a release PR exists yet.
9791                        self.state
9792                            .event("bump", format!("release bump skipped: {e:#}"));
9793                    }
9794                    // Independent of the bump, and best-effort in the same way:
9795                    // findings the merge left open become follow-up tasks.
9796                    if self.state.status == RunStatus::Merged {
9797                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9798                    }
9799                    self.state.save()?;
9800                }
9801                Err(e) => {
9802                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9803                        return Err(e);
9804                    }
9805                    self.state.status = RunStatus::Blocked;
9806                    self.state.event("land", format!("gave up: {e}"));
9807                    self.state.save()?;
9808                }
9809            }
9810            break;
9811        }
9812        Ok(())
9813    }
9814
9815    /// One repair cycle stays under the existing driver/task claim. An
9816    /// interrupted fixer is rolled back and held rather than run twice.
9817    async fn repair_merge_review(&mut self) -> Result<bool> {
9818        use crate::run::MergeRepairPhase as Phase;
9819        crate::pr_head::ensure_owner(&self.state)?;
9820        let i = self.state.merge_repairs.len() - 1;
9821        let request = self.state.merge_repairs[i].clone();
9822        if request.phase == Phase::Fixing {
9823            let restored = match self.state.winner().cloned() {
9824                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9825                None => Err(anyhow::anyhow!("repair has no winning branch")),
9826            };
9827            let why = match restored {
9828                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9829                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9830                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9831            };
9832            self.hold_merge_repair(i, why)?;
9833            return Ok(false);
9834        }
9835        // A completed repair gate is a publication checkpoint. On reentry,
9836        // reject external changes before the ordinary gate path can replace
9837        // that evidence by verifying a new head. Requested repairs still
9838        // start their own review generation below.
9839        if request.phase == Phase::Reviewing && self.state.gate_ran {
9840            let winner = self
9841                .state
9842                .winner()
9843                .context("repair has no winning branch")?;
9844            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9845            if self.state.gate_head.as_deref() != Some(head.as_str())
9846                || !git::is_clean(&winner.worktree).await?
9847            {
9848                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9849                return Ok(false);
9850            }
9851        }
9852        let failure = if i >= self.state.config.graph.land_rounds
9853            || self.state.config.graph.review_rounds == 0
9854        {
9855            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9856        } else {
9857            None
9858        };
9859        if let Some(why) = failure {
9860            self.hold_merge_repair(i, why)?;
9861            return Ok(false);
9862        }
9863        if request.phase == Phase::Requested {
9864            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9865                Ok(pr) => pr,
9866                Err(e) => {
9867                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9868                    return Ok(false);
9869                }
9870            };
9871            self.state.merge_repairs[i].phase = Phase::Fixing;
9872            self.state.land_approval = None;
9873            self.state.land_armed_head = None;
9874            self.state.save()?;
9875            // Serialize the exact records, including finding ids/details,
9876            // reconsideration reasons, final votes and verification output.
9877            let evidence = serde_json::to_string_pretty(&request)?;
9878            let reason = format!(
9879                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9880            );
9881            let budget = self.state.config.graph.land_rounds;
9882            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9883            match result {
9884                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9885                Ok(land::Fixed::Deferred) => {
9886                    // An unavailable fixer did not spend this repair request.
9887                    // Restore partial fallback work before retrying it, as for
9888                    // an interrupted repair, without closing the request.
9889                    let worktree = self
9890                        .state
9891                        .winner()
9892                        .context("repair has no winning branch")?
9893                        .worktree
9894                        .clone();
9895                    self.graft_restore(&worktree, &request.head).await?;
9896                    self.state.merge_repairs[i].phase = Phase::Requested;
9897                    self.stop_for_resource()?;
9898                    return Ok(false);
9899                }
9900                Ok(land::Fixed::Failed(why)) => {
9901                    self.hold_merge_repair(
9902                        i,
9903                        format!(
9904                            "repair failed: {why}; inspect the branch and run an explicit review"
9905                        ),
9906                    )?;
9907                    return Ok(false);
9908                }
9909                Err(e) => {
9910                    self.hold_merge_repair(
9911                        i,
9912                        format!(
9913                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9914                        ),
9915                    )?;
9916                    return Ok(false);
9917                }
9918            }
9919            // Even no change gets a real new reviewer wave. The previous
9920            // evidence remains in the request; a former clean/concluded round
9921            // cannot short-circuit the new review or gate.
9922            self.begin_merge_repair_review(i)?;
9923        }
9924        self.review_loop().await?;
9925        self.gate().await?;
9926        if self.resource_halt()? || self.state.parked {
9927            return Ok(false);
9928        }
9929        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9930            != Some(RunStatus::Gating)
9931            || !self.state.gate_allows_progress()
9932        {
9933            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9934            return Ok(false);
9935        }
9936        // Review and gate may have added commits after the initial repair.
9937        // Publish exactly their final head to the existing PR, with a lease.
9938        let winner = self
9939            .state
9940            .winner()
9941            .cloned()
9942            .context("repair has no winning branch")?;
9943        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9944        if self.state.gate_head.as_deref() != Some(after.as_str())
9945            || !git::is_clean(&winner.worktree).await?
9946        {
9947            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9948            return Ok(false);
9949        }
9950        let expected = self
9951            .state
9952            .pr_pushed_head
9953            .as_deref()
9954            .unwrap_or(&request.head)
9955            .to_owned();
9956        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9957        // Push the checked commit itself: a local ref moving after the check
9958        // must not change which commit the forge receives.
9959        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9960            return Ok(false);
9961        }
9962        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9963        let refspec = format!("{after}:refs/heads/{public}");
9964        let push = git::git_raw(
9965            &winner.worktree,
9966            &["push", &lease, &self.state.config.merge.remote, &refspec],
9967        )
9968        .await;
9969        match push {
9970            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9971            Ok(out) => {
9972                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9973                return Ok(false);
9974            }
9975            Err(e) => {
9976                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9977                return Ok(false);
9978            }
9979        }
9980        self.state.merge_repairs[i].phase = Phase::Complete;
9981        self.state.status = RunStatus::Landing;
9982        self.state.event(
9983            "repair",
9984            "new review and gate completed; fresh owner merge decision required",
9985        );
9986        self.state.save()?;
9987        Ok(true)
9988    }
9989
9990    fn merge_repair_artifacts(&self) -> PathBuf {
9991        let artifacts = agent::artifacts_dir(&self.state.dir());
9992        match self
9993            .state
9994            .merge_repairs
9995            .last()
9996            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
9997        {
9998            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
9999            None => artifacts,
10000        }
10001    }
10002
10003    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10004        if let Some(e2e) = self.state.head_e2e.take() {
10005            self.state
10006                .verification_history
10007                .push(crate::run::VerificationEvidence {
10008                    superseded_by: None,
10009                    archived_at: Some(Timestamp::now()),
10010                    fixes_spent: Some(self.state.gate_fixes.len()),
10011                    gate_head: None,
10012                    gate_ran: false,
10013                    gate: Vec::new(),
10014                    e2e: Some(e2e),
10015                });
10016        }
10017        self.state.reviews.clear();
10018        self.state.gate.clear();
10019        self.state.gate_ran = false;
10020        self.state.gate_head = None;
10021        self.state.gate_fixes.clear();
10022        self.state.gate_repair_e2e = None;
10023        self.state.pre_gate.clear();
10024        self.state.pre_gate_commit = None;
10025        self.state.contested_handoff = None;
10026        self.state.land_reruns = None;
10027        self.state.land_approval = None;
10028        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10029        self.state.status = RunStatus::Reviewing;
10030        self.state.save()
10031    }
10032
10033    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10034        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10035        self.state.merge_repairs[index].failure = Some(why.clone());
10036        self.state.status = RunStatus::Blocked;
10037        self.state.event("repair", why);
10038        self.state.save()
10039    }
10040
10041    // -------------------------------------------------------------- helpers
10042
10043    /// Fetch or create a seat, keeping its conversation across nodes.
10044    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10045        if let Some(existing) = self.state.seats.get(key)
10046            && existing.agent == agent
10047        {
10048            return existing.clone();
10049        }
10050        // A seat that changes agent mints its session id from the agent too,
10051        // like a handover: the old agent's uuid is already taken by the CLI.
10052        let fresh = if self.state.seats.contains_key(key) {
10053            handover_seat(key, agent, self.state.next_seat_seed())
10054        } else {
10055            SeatState::new(key, agent, self.state.seed)
10056        };
10057        self.state.seats.insert(key.to_owned(), fresh.clone());
10058        fresh
10059    }
10060
10061    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10062    /// seat to another roster agent, in which case that agent. Nodes that
10063    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10064    /// reconsideration) must keep talking to whoever answered it, not slip
10065    /// back to the agent that failed it.
10066    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10067        match self.state.seats.get(key) {
10068            Some(s) if s.agent != spec.id => {
10069                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10070            }
10071            _ => spec,
10072        }
10073    }
10074
10075    /// A candidate rendered for judging, with the leak policy applied.
10076    fn view(&self, c: &Candidate) -> CandidateView {
10077        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10078            .unwrap_or_default();
10079        let (patch, _) = blind::sanitize_patch(
10080            &format!("candidate {} patch", c.label),
10081            &raw,
10082            &self.state.config.blind,
10083        );
10084        CandidateView {
10085            label: c.label,
10086            branch: c.branch.clone(),
10087            summary: c.summary.clone(),
10088            stat: c.stat.clone(),
10089            patch,
10090        }
10091    }
10092
10093    /// The full candidate set as prompt text, for seats with no live session.
10094    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10095        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10096        prompt::judge(
10097            "(see above)",
10098            &views,
10099            self.roles.judges.len(),
10100            base_short,
10101            "en",
10102        )
10103    }
10104
10105    /// The final-vote prompt with everything a seat that has no session of its
10106    /// own needs: the candidates, the seat's own ranking and reasons, and the
10107    /// anonymised deliberation it took part in (only when there was one, so a
10108    /// handed-over seat never sees more than the seat it replaces did). The
10109    /// `Final vote` heading stays first.
10110    fn vote_prompt_full(
10111        &self,
10112        j: usize,
10113        labels: &[char],
10114        language: &str,
10115        candidates: &[Candidate],
10116        base_short: &str,
10117    ) -> String {
10118        let mut text = format!(
10119            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10120            prompt::final_vote(labels, language),
10121            self.state.instruction,
10122            self.candidate_block(candidates, base_short)
10123        );
10124        if let Some(own) = self
10125            .state
10126            .judgements
10127            .get(j)
10128            .filter(|r| !r.ranking.is_empty())
10129        {
10130            let reasons = own
10131                .reasons
10132                .iter()
10133                .map(|(k, v)| format!("- {k}: {v}"))
10134                .collect::<Vec<_>>()
10135                .join("\n");
10136            text.push_str(&format!(
10137                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10138                own.ranking.iter().collect::<String>(),
10139                if reasons.is_empty() {
10140                    ""
10141                } else {
10142                    ", because:\n"
10143                }
10144            ));
10145        }
10146        if !self.state.deliberation.is_empty() {
10147            text.push_str("\n# What was argued before this vote\n");
10148            for t in self.transcript(&[], j) {
10149                text.push_str(&format!(
10150                    "\n## {}{}\n\n{}\n",
10151                    t.who,
10152                    if t.is_self { " (you)" } else { "" },
10153                    t.body.trim()
10154                ));
10155            }
10156        }
10157        text
10158    }
10159
10160    /// Anonymised transcript for judge `self_idx`.
10161    ///
10162    /// The initial rankings are always the opening statements. Seeding them
10163    /// only when no turn had been taken yet meant every judge after the first
10164    /// argued against a single voice instead of against the actual split — the
10165    /// disagreement is the information, so it is always on the table.
10166    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10167        let mut turns = Vec::new();
10168        for j in &self.state.judgements {
10169            if j.ranking.is_empty() {
10170                continue;
10171            }
10172            let reasons = j
10173                .reasons
10174                .iter()
10175                .map(|(k, v)| format!("- {k}: {v}"))
10176                .collect::<Vec<_>>()
10177                .join("\n");
10178            turns.push(Turn {
10179                who: format!("Judge {} (opening ranking)", j.judge),
10180                is_self: j.judge == self_idx + 1,
10181                body: format!(
10182                    "Ranked {}{}{reasons}",
10183                    j.ranking.iter().collect::<String>(),
10184                    if reasons.is_empty() {
10185                        ""
10186                    } else {
10187                        ", because:\n"
10188                    }
10189                ),
10190            });
10191        }
10192        for t in self
10193            .state
10194            .deliberation
10195            .iter()
10196            .flat_map(|r| r.turns.iter())
10197            .chain(current)
10198        {
10199            turns.push(Turn {
10200                who: format!("Judge {}", t.judge),
10201                is_self: t.judge == self_idx + 1,
10202                body: t.body.clone(),
10203            });
10204        }
10205        turns
10206    }
10207}
10208
10209/// Does this seat still hold the context a follow-up prompt would rely on?
10210fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10211    agent::has_session(spec.kind, seat, sessions)
10212}
10213
10214/// The next entry in `roster` after `start`, never wrapping back to the
10215/// front, whose id is not in `tried` yet.
10216///
10217/// Starts one past `start` rather than at the front of `roster`: `start` is
10218/// the seat's own original position, and a seat whose candidate slot already
10219/// sits on the roster's second entry must fall through to the third next, not
10220/// restart at the first — which is very likely a different candidate's own
10221/// agent already. Never wraps back past `start`, for the same reason: an
10222/// entry earlier in the roster than the seat's own position is almost
10223/// certainly some *other* candidate slot's own agent, and once the tail of
10224/// the roster is exhausted there are no more untried agents for *this* seat
10225/// to fall through to — the caller's fallback chain ends there, exactly as
10226/// "no further untried agents remain in the list for that seat" asks for.
10227///
10228/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10229/// the same id twice (an operator's `roles.implementers` typo, or a
10230/// `[[agents]]` list reused across roles) must not let
10231/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10232/// over `roster` either finds an untried id or runs out, so this always
10233/// terminates regardless of duplicates.
10234fn next_untried_in_roster<'a>(
10235    roster: &'a [AgentSpec],
10236    start: usize,
10237    tried: &BTreeSet<String>,
10238) -> Option<&'a AgentSpec> {
10239    roster
10240        .get(start + 1..)?
10241        .iter()
10242        .find(|s| !tried.contains(&s.id))
10243}
10244
10245/// The successor for a seat, shared by all four seat kinds (implement, judge,
10246/// review, advise). `others` holds the ids that *currently* occupy the other
10247/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10248/// sees them), so roster entries beyond the seat count act as spares: a failed
10249/// seat goes to an agent no other seat holds whenever the roster permits.
10250///
10251/// `carried` is `Some` only for the review loop's carried failure history.
10252/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10253/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10254/// whole roster: untried and not another seat's occupant; (3) the plain walk
10255/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10256/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10257/// a better outcome than an empty seat, and it keeps the answer to "is there
10258/// a successor at all" exactly what it was before occupants were considered,
10259/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10260/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10261/// carried failure is a cheaper bet than doubling an agent on the panel, and
10262/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10263/// the same round are handled one at a time, so a seat later in the batch
10264/// sees an earlier one's new occupant but not yet its own freed agent.
10265fn pick_successor<'a>(
10266    roster: &'a [AgentSpec],
10267    start: usize,
10268    tried: &BTreeSet<String>,
10269    carried: Option<&BTreeSet<String>>,
10270    others: &BTreeSet<String>,
10271) -> Option<&'a AgentSpec> {
10272    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10273    roster
10274        .get(start + 1..)
10275        .and_then(|tail| {
10276            tail.iter()
10277                .filter(free)
10278                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10279        })
10280        .or_else(|| {
10281            carried
10282                .is_some()
10283                .then(|| roster.iter().find(free))
10284                .flatten()
10285        })
10286        .or_else(|| match carried {
10287            Some(c) => next_for_seat(roster, start, tried, c),
10288            None => next_untried_in_roster(roster, start, tried),
10289        })
10290}
10291
10292/// The next agent for a seat that carries its failure history across rounds
10293/// (the review loop). `round_tried` is this round's own bound and starts
10294/// empty every round; `carried_failed` only decides priority.
10295///
10296/// First: an id walking forward from `start`, never wrapping, that is neither
10297/// tried this round nor failed in an earlier one. Only when that is exhausted
10298/// does it rescue: the first roster id (in roster order, so this one *does*
10299/// look before `start`) not yet tried this round, which is by then a carried
10300/// failure. Each id is rescued at most once per round, so it cannot loop.
10301fn next_for_seat<'a>(
10302    roster: &'a [AgentSpec],
10303    start: usize,
10304    round_tried: &BTreeSet<String>,
10305    carried_failed: &BTreeSet<String>,
10306) -> Option<&'a AgentSpec> {
10307    roster
10308        .get(start + 1..)?
10309        .iter()
10310        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10311        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10312}
10313
10314/// Where a reviewer seat starts a round: the agent that last answered it when
10315/// it is still on the roster and not marked failed, else the spec's own agent
10316/// unless it failed, else the next roster agent that has not failed, else the
10317/// spec's own agent again (the whole roster failed). Ids no longer on the
10318/// roster are ignored. An empty roster has no handover, so the spec stands.
10319fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10320    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10321        return spec;
10322    };
10323    let ok = |id: &str| !h.failed.contains(id);
10324    if let Some(last) = h.last_ok.as_deref()
10325        && ok(last)
10326        && let Some(s) = roster.iter().find(|s| s.id == last)
10327    {
10328        return s.clone();
10329    }
10330    if ok(&spec.id) {
10331        return spec;
10332    }
10333    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10334    roster
10335        .get(from + 1..)
10336        .into_iter()
10337        .flatten()
10338        .chain(roster.iter())
10339        .find(|s| ok(&s.id))
10340        .cloned()
10341        .unwrap_or(spec)
10342}
10343
10344/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10345/// seed so a CLI that mints its session id up front (`--session-id`) never
10346/// reuses the uuid the previous agent already opened under the same seat key.
10347pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10348    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10349}
10350
10351/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10352/// so the classifier below and the code that builds the message cannot drift.
10353const TIMED_OUT: &str = "timed out";
10354
10355impl FailClass {
10356    /// `None` for an answer; otherwise how the turn failed.
10357    fn of(out: &AgentOutcome) -> Option<Self> {
10358        match out {
10359            AgentOutcome::Ok(_) => None,
10360            AgentOutcome::Quota(_) => Some(Self::Quota),
10361            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10362            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10363            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10364            // Not a failure of the agent. Callers check for it first; `None`
10365            // here means "no class to hand over on".
10366            AgentOutcome::ResourceBlocked(_) => None,
10367        }
10368    }
10369
10370    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10371    fn stem_word(&self) -> &'static str {
10372        match self {
10373            Self::Quota => "quota",
10374            _ => "handover",
10375        }
10376    }
10377}
10378
10379/// The message's first line with its variable parts removed — digit runs and
10380/// path-like tokens — so "exit code 2" and "exit code 7" read
10381/// as one kind of failure.
10382fn failure_signature(msg: &str) -> String {
10383    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10384    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10385    // the historical machine key while the process diagnosis uses readable prose.
10386    if let Some(code) = line
10387        .strip_prefix("process exited with no usable output (exit code ")
10388        .and_then(|s| s.strip_suffix(')'))
10389        .and_then(|s| s.parse::<i32>().ok())
10390    {
10391        line = format!("exited with some({code}) and no usable output");
10392    } else if matches!(
10393        line.as_str(),
10394        "process exited with no usable output (no exit code reported)"
10395            | "process exited with no usable output (exit code unavailable)"
10396    ) {
10397        line = "exited with none and no usable output".to_owned();
10398    }
10399    let mut out = Vec::new();
10400    for word in line.split_whitespace() {
10401        if word.contains('/') || word.contains('\\') {
10402            out.push("<path>".to_owned());
10403            continue;
10404        }
10405        let mut w = String::new();
10406        let mut in_digits = false;
10407        for c in word.chars() {
10408            if c.is_ascii_digit() {
10409                if !in_digits {
10410                    w.push('#');
10411                }
10412                in_digits = true;
10413            } else {
10414                in_digits = false;
10415                w.push(c);
10416            }
10417        }
10418        out.push(w);
10419    }
10420    out.join(" ").chars().take(120).collect()
10421}
10422
10423/// Whether a seat that just failed with `cur` may go to the next roster agent.
10424/// A quota or a timeout always may. Any other failure may not when the agent
10425/// before it failed the same way: an error the prompt causes would otherwise
10426/// walk the whole roster. `prev` is the class of the immediately preceding
10427/// agent's failure, so a quota or timeout in between breaks the run of
10428/// identical failures by itself.
10429fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10430    match cur {
10431        FailClass::Quota | FailClass::Timeout => true,
10432        FailClass::Other(_) => prev != Some(cur),
10433    }
10434}
10435
10436/// A short human reason for a failed outcome, for the handover record.
10437fn fail_reason(out: &AgentOutcome) -> String {
10438    match out {
10439        AgentOutcome::Ok(_) => String::new(),
10440        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10441        AgentOutcome::Dropped(o) => format!(
10442            "the CLI dropped the stream ({})",
10443            o.dropped
10444                .as_ref()
10445                .map(|d| d.why.as_str())
10446                .unwrap_or("it ended without delivering its answer")
10447        ),
10448        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10449            e.lines().next().unwrap_or("").chars().take(160).collect()
10450        }
10451    }
10452}
10453
10454/// Note one handover in the run: the structured record and, in the timeline,
10455/// the sentence a person reads. A quota keeps the wording it always had.
10456pub(crate) fn record_handover(
10457    state: &mut RunState,
10458    node: &str,
10459    seat: &str,
10460    from: &str,
10461    to: &str,
10462    class: &FailClass,
10463    reason: &str,
10464) {
10465    let message = if *class == FailClass::Quota {
10466        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10467    } else {
10468        format!("{seat}: handed over {from} -> {to} ({reason})")
10469    };
10470    state.event(node, message);
10471    state.handovers.push(Handover {
10472        at: Timestamp::now(),
10473        node: node.to_owned(),
10474        seat: seat.to_owned(),
10475        from: from.to_owned(),
10476        to: to.to_owned(),
10477        reason: reason.to_owned(),
10478    });
10479}
10480
10481/// Did this reply report running a command whose own CLI never confirmed an
10482/// exit status?
10483///
10484/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10485/// command *finished* (see that type's own doc), so this can only be `true`
10486/// for a command whose completion event carried no readable exit code — not
10487/// for one that simply is not mentioned at all. That is the one signal this
10488/// crate can read, from the same record `state.jobs` renders, about a reply
10489/// standing next to work its own CLI cannot vouch for finishing; it is
10490/// deliberately not a check on the exit code's *value* (a fixer legitimately
10491/// runs a command that fails mid-iteration before it succeeds) and not a
10492/// guess at a command still running in the background (which emits no event
10493/// at all, and so leaves no evidence here to find).
10494fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10495    commands.iter().any(|c| c.exit_code.is_none())
10496}
10497
10498/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10499/// trusted as a verified no-op — the adoption guard's own text-level half.
10500///
10501/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10502/// timed out): a marker only earns the benefit of the doubt from a turn the
10503/// CLI itself vouches for finishing properly, the same house style
10504/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10505/// *fix* report to for `commands`. A candidate that timed out, exited
10506/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10507/// is, whatever prose it wrote — this returns `None` before it ever looks at
10508/// `text`. The remaining guards (the tree really is empty, the evidence is
10509/// non-empty) are the caller's: this only reads what the reply *claimed*.
10510fn verified_noop_claim(
10511    usable: bool,
10512    commands: &[agent::CommandEvidence],
10513    text: &str,
10514) -> Option<String> {
10515    (usable && !has_unconfirmed_command(commands))
10516        .then(|| verdict::verified_noop(text))
10517        .flatten()
10518}
10519
10520fn short(commit: &str) -> String {
10521    commit.chars().take(7).collect()
10522}
10523
10524fn make_executable(path: &Path) -> Result<()> {
10525    #[cfg(unix)]
10526    {
10527        use std::os::unix::fs::PermissionsExt as _;
10528        let mut perms = std::fs::metadata(path)?.permissions();
10529        perms.set_mode(0o755);
10530        std::fs::set_permissions(path, perms)?;
10531    }
10532    #[cfg(not(unix))]
10533    {
10534        let _ = path;
10535    }
10536    Ok(())
10537}
10538
10539/// What every seat in one batch shares: where the answers are attributed, the
10540/// prompt overlay they inherit, and the build cache they are told to use.
10541///
10542/// A struct rather than four more parameters: `wave` also needs the run's
10543/// state (to record who is answering right now) and the attempt number, and
10544/// eight positional arguments is both unreadable and a clippy error.
10545struct WaveCtx<'a> {
10546    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10547    /// paid for it.
10548    run: &'a str,
10549    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10550    node: &'a str,
10551    prompts: &'a Prompts,
10552    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10553    cache: Option<&'a Path>,
10554    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10555    /// `JobRecord::round`. `None` for every other node.
10556    round: Option<usize>,
10557    /// Carry each seat's failed-agent history across waves (the review loop
10558    /// only): start-of-round priority and handover choice read
10559    /// [`RunState::seat_history`], and every answer or failure writes it.
10560    carry_seats: bool,
10561    /// The run's park / interrupt requests, so a wait for a shared resource
10562    /// can be abandoned the moment the operator asks to park.
10563    pauses: Option<(&'a Pause, &'a Pause)>,
10564}
10565
10566/// Shared scheduling policy for operations and spawned seat invocations.
10567fn repository_wait(
10568    state: &RunState,
10569    pauses: Option<(&Pause, &Pause)>,
10570) -> crate::repo_lock::WaitPolicy {
10571    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10572    let inherited = crate::repo_lock::WaitPolicy::current();
10573    crate::repo_lock::WaitPolicy::new(
10574        Duration::from_secs(state.config.graph.timeout_resource_wait),
10575        move || {
10576            inherited.cancelled()
10577                || pauses
10578                    .as_ref()
10579                    .is_some_and(|(a, b)| a.parked() || b.parked())
10580        },
10581    )
10582}
10583
10584fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10585    if error.is::<crate::repo_lock::ResourceBlocked>() {
10586        state.resource_block = Some(crate::run::ResourceBlock {
10587            cause: crate::run::ResourceBlockCause::Other,
10588            node: "repository".into(),
10589            kind: if error.to_string().contains("disk pressure") {
10590                "disk pressure"
10591            } else if error.to_string().contains("disk admission") {
10592                "disk admission"
10593            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10594                "incremental cache admission"
10595            } else {
10596                "repository coordination"
10597            }
10598            .into(),
10599            detail: error.to_string(),
10600            at: Timestamp::now(),
10601        });
10602        state.event("resource", error.to_string());
10603    }
10604}
10605
10606async fn coordinate_operation(
10607    state: &mut RunState,
10608    cwd: &Path,
10609    wait: &crate::repo_lock::WaitPolicy,
10610) -> Result<Option<crate::repo_lock::SeatLease>> {
10611    let result =
10612        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10613            .await;
10614    if let Err(e) = &result {
10615        record_repository_block(state, e);
10616    }
10617    result
10618}
10619
10620/// Run one job, honouring the parallelism budget.
10621async fn run_one(
10622    job: SeatJob,
10623    sem: Arc<Semaphore>,
10624    ctx: &WaveCtx<'_>,
10625    state: &mut RunState,
10626    attempt: usize,
10627) -> (SeatState, AgentOutcome) {
10628    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10629        .await
10630        .pop()
10631        .expect("one job in, one result out");
10632    (seat, out)
10633}
10634
10635/// Run every job concurrently, capped by the semaphore, preserving order.
10636///
10637/// Every seat in the batch is recorded into [`RunState::active`] before the
10638/// wave starts and cleared as each answer lands, so the run's own record says
10639/// who is still being waited on rather than only who finished.
10640async fn wave(
10641    jobs: Vec<SeatJob>,
10642    sem: Arc<Semaphore>,
10643    ctx: &WaveCtx<'_>,
10644    state: &mut RunState,
10645    attempt: usize,
10646) -> Vec<(usize, SeatState, AgentOutcome)> {
10647    let WaveCtx {
10648        run,
10649        node,
10650        prompts,
10651        cache,
10652        round,
10653        carry_seats: _,
10654        pauses,
10655    } = *ctx;
10656    for job in &jobs {
10657        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10658    }
10659    if let Err(e) = state.save() {
10660        // A failed persist of "who is answering right now" must not abort the
10661        // wave: the seats are already being asked, and the alternative is
10662        // losing the answers to save a status line nobody may even be
10663        // watching.
10664        tracing::warn!("could not persist in-progress seats: {e:#}");
10665    }
10666    // Hold the shared build cache's lease for the whole batch, not per job:
10667    // several candidates (an implement wave) or a fixer legitimately share
10668    // one cache concurrently within this run, and that stays untouched — a
10669    // single lease taken once for the whole wave and released once it is
10670    // done is what stops a *different* borrower (another run's own wave, its
10671    // e2e/gate, a human's `magi review`) from interleaving a build into the
10672    // same directory while this one is in flight. Best-effort, not
10673    // all-or-nothing: a wave that cannot get the lease within its own
10674    // longest job's budget still runs — an hour of paid implementer calls is
10675    // not thrown away over cache contention — but every write-allowed seat
10676    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10677    // below), the same fallback a read-only seat always gets, rather than
10678    // building into a directory this run was never granted. The identity
10679    // record is still invalidated below either way, so the next tracked
10680    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10681    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10682    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10683    // never borrowed from a job's agent budget (and never charged to it either,
10684    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10685    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10686    let cache_guard = if let Some(cache_dir) = cache {
10687        if jobs_had_a_writer {
10688            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10689            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10690            match acquire_cache_lease(
10691                state,
10692                cache_dir,
10693                &owner,
10694                resource_limit,
10695                node,
10696                node,
10697                &cancel,
10698            )
10699            .await
10700            {
10701                Ok(Some(g)) => Some(g),
10702                // Parking: leave without spawning anything. The run parks at
10703                // the next boundary and the seats are asked afresh on resume.
10704                Ok(None) => {
10705                    if let Some(block) = &mut state.resource_block {
10706                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10707                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10708                        };
10709                    }
10710                    return block_wave(
10711                        jobs,
10712                        state,
10713                        node,
10714                        attempt,
10715                        "parked while waiting for the shared build cache",
10716                    );
10717                }
10718                // An unreadable lease is not contention: keep the old
10719                // fallback (no shared cache for this wave's seats).
10720                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10721                Err(e) => {
10722                    // Writing seats may not build into a cache this run does
10723                    // not hold, and an agent that cannot be given one is not
10724                    // asked at all: the run stops resumably instead.
10725                    let why = format!("{e:#}");
10726                    state.event(
10727                        "resource",
10728                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10729                    );
10730                    return block_wave(jobs, state, node, attempt, &why);
10731                }
10732            }
10733        } else {
10734            None
10735        }
10736    } else {
10737        None
10738    };
10739    // Every job keeps its whole budget: the wait above was not the agent's.
10740    // Restart the seats' clocks so "started" means "being spawned now".
10741    state.seats_spawning(node, attempt);
10742    if let Err(e) = state.save() {
10743        tracing::warn!("could not persist the start of a wave: {e:#}");
10744    }
10745    // Queueing behind this wave's own seats for a parallelism slot is normal
10746    // (each of them is bounded by its own agent timeout); only a wait beyond
10747    // all of them plus the resource limit means the slots never came back.
10748    let slot_limit = resource_limit
10749        + jobs
10750            .iter()
10751            .map(|j| j.timeout)
10752            .max()
10753            .unwrap_or_default()
10754            .saturating_mul(jobs.len() as u32);
10755    let mut set = tokio::task::JoinSet::new();
10756    // Seats announce themselves here the moment they hold a parallelism
10757    // permit, i.e. just before their CLI is spawned.
10758    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10759    let overlay = prompts.overlay(node);
10760    let repository_wait = repository_wait(state, pauses);
10761    for (i, mut job) in jobs.into_iter().enumerate() {
10762        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10763        if cache.is_some() {
10764            job.prompt.push('\n');
10765            job.prompt
10766                .push_str(&prompt::build_cache_note(node, job.allow_write));
10767        }
10768        let sem = Arc::clone(&sem);
10769        let run = run.to_owned();
10770        let node = node.to_owned();
10771        // Only implementers were told about the task's attachments, so only
10772        // their seats get the directory widened for reading.
10773        let attachments = if node == "implement" {
10774            state.attachments.clone()
10775        } else {
10776            Vec::new()
10777        };
10778        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10779        // `prompt::build_cache_note`'s doc for why setting it anyway is
10780        // exactly how a sandboxed reviewer's write refusal got reported as a
10781        // defect in the patch, not a property of its own seat. And a
10782        // write-allowed one is handed it only when the lease above was
10783        // actually acquired: a wave that could not get it (`cache_guard` is
10784        // `None`, see its own comment) must not send seats to build into a
10785        // directory this run does not hold - that is the exact concurrent,
10786        // unmanaged-write race this module exists to prevent, not something
10787        // "proceeding anyway" is allowed to reintroduce.
10788        let cache = cache
10789            .filter(|_| job.allow_write && cache_guard.is_some())
10790            .map(Path::to_path_buf);
10791        let spawned_tx = spawned_tx.clone();
10792        let repository_wait = repository_wait.clone();
10793        let coordination = state.config.coordination.clone();
10794        let fetched_base = state.fetched_base.clone();
10795        set.spawn(async move {
10796            // A wait for a parallelism slot is scheduling, not the agent: it
10797            // is bounded by its own limit and ends as a resource block, never
10798            // as a timeout of the agent that was not started.
10799            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10800                Ok(p) => p,
10801                Err(_) => {
10802                    return (
10803                        i,
10804                        job.seat,
10805                        AgentOutcome::ResourceBlocked(format!(
10806                            "no parallel slot became free within {}s",
10807                            slot_limit.as_secs()
10808                        )),
10809                        false,
10810                    );
10811                }
10812            };
10813            spawned_tx.send(job.seat.key.clone()).ok();
10814            let mut seat = job.seat;
10815            let out = repository_wait
10816                .scope(agent::invoke(
10817                    &job.spec,
10818                    &mut seat,
10819                    &Invocation {
10820                        coordination: coordination.as_ref(),
10821                        fetched_base: fetched_base.as_ref(),
10822                        cwd: &job.cwd,
10823                        prompt: &job.prompt,
10824                        timeout: job.timeout,
10825                        allow_write: job.allow_write,
10826                        sessions: job.sessions,
10827                        artifacts: &job.artifacts,
10828                        stem: &job.stem,
10829                        run: &run,
10830                        node: &node,
10831                        cache_dir: cache.as_deref(),
10832                        attachments: &attachments,
10833                        writable: &[],
10834                    },
10835                ))
10836                .await;
10837            let interrupted = out
10838                .as_ref()
10839                .err()
10840                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10841            let out = match out {
10842                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10843                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10844                // Billed work the CLI failed to hand over is not an ordinary
10845                // failure, but its text is the CLI's raw error JSON, not an
10846                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10847                // read it as one by forgetting to check. `usable()` is always
10848                // false here (dropped implies an empty response), so this has
10849                // to be checked before the catch-all `Failed` below or the
10850                // one shape this exists for is lost with the rest.
10851                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10852                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10853                Ok(o) => AgentOutcome::Failed(format!(
10854                    "process exited with no usable output ({})",
10855                    crate::notices::exit_status(o.exit_code)
10856                )),
10857                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10858                    AgentOutcome::ResourceBlocked(e.to_string())
10859                }
10860                Err(e) => AgentOutcome::Failed(e.to_string()),
10861            };
10862            (i, seat, out, interrupted)
10863        });
10864    }
10865    drop(spawned_tx);
10866    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10867    loop {
10868        let joined = tokio::select! {
10869            Some(key) = spawned_rx.recv() => {
10870                // The clock of this seat starts now, not when it queued.
10871                state.seat_spawned(&key);
10872                if let Err(e) = state.save() {
10873                    tracing::warn!("could not persist a seat's start: {e:#}");
10874                }
10875                continue;
10876            }
10877            joined = set.join_next() => joined,
10878        };
10879        let Some(joined) = joined else {
10880            break;
10881        };
10882        let (i, seat, out, interrupted) = match joined {
10883            Ok(v) => v,
10884            // No seat to clear: a panicked task never reported which one it
10885            // was. The defensive sweep below this loop is what stops that
10886            // seat's `active` entry from surviving forever.
10887            Err(e) => {
10888                tracing::error!("agent task panicked: {e}");
10889                continue;
10890            }
10891        };
10892        state.seat_finished(&seat.key);
10893        if interrupted
10894            && node == "implement"
10895            && let Some(label) = state
10896                .candidates
10897                .iter()
10898                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10899                .map(|candidate| candidate.label)
10900        {
10901            state.candidate_preservation.insert(
10902                label,
10903                "interrupted implementation awaiting persistence".into(),
10904            );
10905            state.interrupted_implementations.insert(label);
10906        }
10907        if let AgentOutcome::ResourceBlocked(why) = &out
10908            && state.resource_block.is_none()
10909        {
10910            state.resource_block = Some(crate::run::ResourceBlock {
10911                cause: crate::run::ResourceBlockCause::Other,
10912                node: node.to_owned(),
10913                kind: if why.contains("disk pressure") {
10914                    "disk pressure"
10915                } else if why.contains("disk admission") {
10916                    "disk admission"
10917                } else if why.contains(crate::incremental::BLOCKED) {
10918                    "incremental cache admission"
10919                } else if why.contains("repository coordination") {
10920                    "repository coordination"
10921                } else {
10922                    "parallel slot"
10923                }
10924                .to_owned(),
10925                detail: why.clone(),
10926                at: Timestamp::now(),
10927            });
10928            let activity = if interrupted {
10929                "agent execution interrupted"
10930            } else {
10931                "no agent was started"
10932            };
10933            state.event("resource", format!("{node}: {activity} - {why}"));
10934        }
10935        record_jobs(state, node, round, &seat.key, &out);
10936        if let Err(e) = state.save() {
10937            tracing::warn!("could not persist a seat's completion: {e:#}");
10938        }
10939        if collected.len() <= i {
10940            collected.resize_with(i + 1, || None);
10941        }
10942        collected[i] = Some((i, seat, out));
10943    }
10944    // Belt-and-braces for the panic branch above: every seat this exact batch
10945    // started shares this `(node, attempt)` pair, and every seat that finished
10946    // normally already cleared itself, so anything left tagged with it here
10947    // can only be a panicked task's leftover. Cleared unconditionally rather
10948    // than left to read as still answering forever.
10949    if state
10950        .active
10951        .values()
10952        .any(|a| a.node == node && a.attempt == attempt)
10953    {
10954        state
10955            .active
10956            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10957        if let Err(e) = state.save() {
10958            tracing::warn!("could not persist the end of a wave: {e:#}");
10959        }
10960    }
10961    // Whether or not the lease above was actually held, several worktrees
10962    // may just have built into the cache with nothing here able to name one
10963    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10964    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10965    // clean it might not have strictly needed; trusting a stale match would
10966    // cost it a wrong answer.
10967    if let Some(cache_dir) = cache
10968        && jobs_had_a_writer
10969    {
10970        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10971    }
10972    if let Some(guard) = cache_guard {
10973        guard.release();
10974    }
10975    collected.into_iter().flatten().collect()
10976}
10977
10978/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10979/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10980/// this is data collection, not the fix-specific completion contract in
10981/// [`Runner::continue_fix_report`], and applies regardless of which node
10982/// asked.
10983///
10984/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
10985/// evidence from; `Failed` does not, and correctly contributes nothing — a
10986/// timeout or crash is not itself evidence about a command the seat may have
10987/// started.
10988fn record_jobs(
10989    state: &mut RunState,
10990    node: &str,
10991    round: Option<usize>,
10992    seat: &str,
10993    out: &AgentOutcome,
10994) {
10995    let commands: &[agent::CommandEvidence] = match out {
10996        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
10997        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
10998    };
10999    let checked_at = Timestamp::now();
11000    for c in commands {
11001        state.jobs.push(JobRecord {
11002            node: node.to_owned(),
11003            round,
11004            seat: seat.to_owned(),
11005            id: c.id.clone(),
11006            description: c.description.clone(),
11007            checked_at,
11008            status: match c.exit_code {
11009                Some(0) => JobStatus::Completed,
11010                Some(_) => JobStatus::Failed,
11011                None => JobStatus::Unknown,
11012            },
11013            exit_code: c.exit_code,
11014            result_summary: c.result_summary.clone(),
11015            source: c.source.clone(),
11016        });
11017    }
11018}
11019
11020/// Is a review round clean, given how many reviewer seats answered against
11021/// how many the round expected?
11022///
11023/// A seat that never answered (timeout, crash, unparsable output) is not a
11024/// seat that read the patch and found nothing — treating it as such is
11025/// exactly the bug this function exists to close. Under the default `block`
11026/// policy a missing seat can never be clean; `warn` still requires the seats
11027/// that *did* answer to have found nothing blocking and verification to be
11028/// green.
11029///
11030/// `quota_missing` narrows that `block` default for exactly one cause of
11031/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11032/// a session limit lifts by the very next round buys nothing — the seat is
11033/// asked again with the same quota — so once every missing seat is accounted
11034/// for by a quota loss (and at least one seat *did* answer, so a decision has
11035/// something to rest on) the round is decided on the panel that could answer,
11036/// same as `warn` would. A panel that lost every seat to quota is not
11037/// decided here: `answered == 0` falls through to the existing `block`
11038/// fallback so a fully collapsed panel still waits rather than landing on no
11039/// review at all.
11040fn round_is_clean(
11041    blocking: usize,
11042    e2e_ok: bool,
11043    answered: usize,
11044    expected: usize,
11045    quota_missing: usize,
11046    policy: IncompleteReviewPolicy,
11047) -> bool {
11048    if blocking != 0 || !e2e_ok {
11049        return false;
11050    }
11051    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11052        return true;
11053    }
11054    answered > 0 && expected - answered <= quota_missing
11055}
11056
11057/// The review loop's own conclusion, derived entirely from its persisted
11058/// round records and the round budget that produced them — never from
11059/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11060/// recomputes the identical answer regardless of what an earlier node in the
11061/// same walk, or a previous walk, did to `status`.
11062///
11063/// `None` while more rounds remain to try, including when review never ran
11064/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11065/// gone clean, or the budget is spent, or the tree has stopped moving (see
11066/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11067///
11068/// - An incomplete panel that raised nothing is missing input, not a
11069///   verified tree — never a hand-off candidate, whatever verification said
11070///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11071/// - Otherwise, green e2e on the last round hands off (see
11072///   [`Runner::stop_reviewing`]); red e2e blocks.
11073///
11074/// A last round whose own verification is still `ResourceBlocked` — magi
11075/// itself never got a command to run, not evidence the patch is broken —
11076/// is neither: this returns `None` for it too, the same as "more rounds
11077/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11078/// handling of that shape) instead of this cheap recomputation guessing a
11079/// verdict a real attempt never produced.
11080pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11081    if max_rounds == 0
11082        || reviews.last().is_some_and(|r| {
11083            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11084        })
11085    {
11086        return Some(RunStatus::Gating);
11087    }
11088    let last = reviews.last()?;
11089    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11090    if reviews.len() < max_rounds && !stagnant {
11091        return None;
11092    }
11093    if last.incomplete() && last.blocking == 0 {
11094        return Some(RunStatus::Blocked);
11095    }
11096    if last.e2e_status() == E2eStatus::ResourceBlocked
11097        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11098    {
11099        return None;
11100    }
11101    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11102        RunStatus::Gating
11103    } else {
11104        RunStatus::Blocked
11105    })
11106}
11107
11108/// How long a re-ask may take, given the budget the first attempt had.
11109///
11110/// A `nudged` retry is a request to restate an answer the seat has already
11111/// worked out: it carries no new work, so it does not deserve the original
11112/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11113/// 133 seconds while a third sat for over ten minutes on a resumed session
11114/// holding 410 KB of prior output - and because the retry had inherited the
11115/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11116/// judging round whose other seats were long finished.
11117///
11118/// A quarter of the budget, with a floor so that a deliberately short timeout
11119/// does not collapse to nothing. A retry that re-sends the whole prompt
11120/// (because the seat kept no context) is the original job again, and keeps the
11121/// original budget.
11122pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11123    if nudged {
11124        (full / 4).max(Duration::from_secs(120)).min(full)
11125    } else {
11126        full
11127    }
11128}
11129
11130/// Run a wave and parse each reply, re-asking the seats whose reply was
11131/// unusable.
11132///
11133/// The re-ask is a nudge rather than the whole prompt again when the seat still
11134/// holds its conversation, which is the difference between a cheap retry and
11135/// paying for the entire candidate set twice.
11136///
11137/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11138/// successor in `roster` is handed to it instead of being re-asked: the
11139/// handover is the retry. A seat with no successor left (a single-agent
11140/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11141/// the asks to one seat in one node number at most
11142/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11143/// is asked once (a dropped stream is nudged first), and only the last agent
11144/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11145/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11146/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11147/// other failure stops the chain when the previous agent failed the same way
11148/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11149/// [`has_context`] is false and the job's own full prompt and full budget are
11150/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11151/// intermediate ones are not losses) and is returned as a failure like any
11152/// other absent seat — the caller decides whether the panel still has a
11153/// quorum. An empty `roster` disables handover: failures are nudged as they
11154/// always were, and a quota is simply lost. A reply that fails to parse or
11155/// validate is the prompt's doing and is only ever nudged, never handed over.
11156///
11157/// The returned [`SeatState`] names the agent that answered (or tried last).
11158#[allow(clippy::too_many_arguments)]
11159async fn ask_json_wave<T>(
11160    jobs: Vec<SeatJob>,
11161    sem: Arc<Semaphore>,
11162    retries: usize,
11163    roster: &[AgentSpec],
11164    ctx: &WaveCtx<'_>,
11165    losses: &mut Vec<QuotaLoss>,
11166    state: &mut RunState,
11167    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11168) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11169where
11170    T: serde::de::DeserializeOwned + Send + 'static,
11171{
11172    ask_wave_with(
11173        jobs,
11174        sem,
11175        retries,
11176        roster,
11177        ctx,
11178        losses,
11179        state,
11180        &|text: &str| {
11181            let v = verdict::extract_json::<T>(text)?;
11182            validate(&v)?;
11183            Ok(v)
11184        },
11185    )
11186    .await
11187}
11188
11189/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11190/// a node whose answer is prose (deliberation) shares the same handover,
11191/// failure classification, quota bookkeeping and bounds instead of a copy.
11192///
11193/// A seat handed to another roster agent is sent the job's `handover` prompt
11194/// (when it has one) rather than `prompt`: the new agent has no session, so a
11195/// resume-style prompt would be incomplete. That holds for the handover ask
11196/// and for every nudge to that agent whose `has_context` is false.
11197#[allow(clippy::too_many_arguments)]
11198async fn ask_wave_with<T>(
11199    jobs: Vec<SeatJob>,
11200    sem: Arc<Semaphore>,
11201    retries: usize,
11202    roster: &[AgentSpec],
11203    ctx: &WaveCtx<'_>,
11204    losses: &mut Vec<QuotaLoss>,
11205    state: &mut RunState,
11206    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11207) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11208where
11209    T: Send + 'static,
11210{
11211    let n = jobs.len();
11212    let originals: Vec<SeatJob> = jobs;
11213    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11214    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11215    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11216    // answer, N once it has gone through N nudges. Read back once this
11217    // returns, so a caller building a history record (`ReviewRecord`) can
11218    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11219    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11220    // field's own doc.
11221    let mut nudges: Vec<usize> = vec![0; n];
11222    // The agent now occupying each seat, the ids it has already been through,
11223    // where in the roster the walk began, the class of the last failure, and
11224    // the stem word of a handover not yet asked (full prompt, full budget).
11225    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11226    let mut tried: Vec<BTreeSet<String>> = specs
11227        .iter()
11228        .map(|s| BTreeSet::from([s.id.clone()]))
11229        .collect();
11230    let starts: Vec<usize> = specs
11231        .iter()
11232        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11233        .collect();
11234    let carry = ctx.carry_seats && !roster.is_empty();
11235    // Carried across rounds: ids that failed the seat earlier, and how the
11236    // last failure went (so a repeat of it is not handed over again).
11237    let carried: Vec<BTreeSet<String>> = originals
11238        .iter()
11239        .map(|j| {
11240            state
11241                .seat_history
11242                .get(&j.seat.key)
11243                .filter(|_| carry)
11244                .map(|h| h.failed.clone())
11245                .unwrap_or_default()
11246        })
11247        .collect();
11248    let mut prev: Vec<Option<FailClass>> = originals
11249        .iter()
11250        .map(|j| {
11251            state
11252                .seat_history
11253                .get(&j.seat.key)
11254                .filter(|_| carry)
11255                .and_then(|h| h.last_fail.clone())
11256        })
11257        .collect();
11258    let next_agent =
11259        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11260            let others: BTreeSet<String> = specs
11261                .iter()
11262                .enumerate()
11263                .filter(|(j, _)| *j != i)
11264                .map(|(_, s)| s.id.clone())
11265                .collect();
11266            pick_successor(
11267                roster,
11268                starts[i],
11269                tried,
11270                carry.then(|| &carried[i]),
11271                &others,
11272            )
11273            .cloned()
11274        };
11275    let mut fresh: Vec<Option<String>> = vec![None; n];
11276    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11277    let mut pending: Vec<usize> = (0..n).collect();
11278
11279    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11280    // asks: an agent with a successor is asked once and handed over, and only
11281    // a seat with no successor spends `retries` nudges on the same agent. This
11282    // only guarantees the loop's own termination whatever those say.
11283    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11284    for round in 0..max_rounds {
11285        if pending.is_empty() {
11286            break;
11287        }
11288        let mut batch = Vec::with_capacity(pending.len());
11289        let mut renudged: Vec<&str> = Vec::new();
11290        for &i in &pending {
11291            let src = &originals[i];
11292            // A seat now held by another agent than the job named has no
11293            // session of its own: it gets the full-context prompt whenever
11294            // it is asked in full (the handover ask, a nudge it cannot
11295            // resume).
11296            let full: &str = match &src.handover {
11297                Some(h) if specs[i].id != src.spec.id => h,
11298                _ => &src.prompt,
11299            };
11300            // The prompt and the budget are one decision: a nudge restates
11301            // finished work, a re-sent prompt redoes it.
11302            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11303                (
11304                    full.to_owned(),
11305                    src.timeout,
11306                    format!("{}-{word}-{}", src.stem, specs[i].id),
11307                )
11308            } else if nudges[i] == 0 {
11309                (full.to_owned(), src.timeout, src.stem.clone())
11310            } else {
11311                renudged.push(src.seat.key.as_str());
11312                let why = done[i]
11313                    .as_ref()
11314                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11315                    .unwrap_or_else(|| "no parsable answer".to_owned());
11316                let nudge = prompt::nudge(&why);
11317                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11318                let prompt = if nudged {
11319                    nudge
11320                } else {
11321                    format!("{full}\n\n---\n\n{nudge}")
11322                };
11323                (
11324                    prompt,
11325                    retry_budget(src.timeout, nudged),
11326                    format!("{}-retry{}", src.stem, nudges[i]),
11327                )
11328            };
11329            batch.push(SeatJob {
11330                spec: specs[i].clone(),
11331                seat: seats[i].clone(),
11332                cwd: src.cwd.clone(),
11333                prompt,
11334                timeout,
11335                allow_write: src.allow_write,
11336                sessions: src.sessions,
11337                artifacts: src.artifacts.clone(),
11338                stem,
11339                handover: None,
11340            });
11341        }
11342
11343        if !renudged.is_empty() {
11344            state.event(
11345                ctx.node,
11346                format!("retry {round}: re-asking {}", renudged.join(", ")),
11347            );
11348        }
11349        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11350        let mut still = Vec::new();
11351        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11352            seats[i] = seat;
11353            // No agent was asked: nothing to hand over, nudge, remember or
11354            // charge. The caller sees `resource_block` and stops the node.
11355            if let AgentOutcome::ResourceBlocked(why) = &out {
11356                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11357                continue;
11358            }
11359            let class = FailClass::of(&out);
11360            // A dropped stream is nudged first (the conversation is still
11361            // there to pick up); only a seat whose nudges are spent hands over.
11362            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11363                && nudges[i] < retries
11364                && !roster.is_empty();
11365            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11366                let next = should_hand_over(prev[i].as_ref(), &cur)
11367                    .then(|| next_agent(i, &tried[i], &specs))
11368                    .flatten();
11369                if carry {
11370                    let h = state
11371                        .seat_history
11372                        .entry(originals[i].seat.key.clone())
11373                        .or_default();
11374                    h.failed.insert(specs[i].id.clone());
11375                    h.last_fail = Some(cur.clone());
11376                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11377                        h.last_ok = None;
11378                    }
11379                    // Saved before the next agent is asked, so a restart in
11380                    // between does not forget who failed.
11381                    if let Err(e) = state.save() {
11382                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11383                    }
11384                }
11385                if let Some(next) = next {
11386                    record_handover(
11387                        state,
11388                        ctx.node,
11389                        &originals[i].seat.key,
11390                        &specs[i].id,
11391                        &next.id,
11392                        &cur,
11393                        &fail_reason(&out),
11394                    );
11395                    tried[i].insert(next.id.clone());
11396                    prev[i] = Some(cur.clone());
11397                    seats[i] =
11398                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11399                    specs[i] = next;
11400                    fresh[i] = Some(cur.stem_word().to_owned());
11401                    nudges[i] = 0;
11402                    done[i] = Some(Err(anyhow::anyhow!(
11403                        "handed over after: {}",
11404                        fail_reason(&out)
11405                    )));
11406                    still.push(i);
11407                    continue;
11408                }
11409            }
11410            if carry
11411                && !nudge_first
11412                && let Some(cur) = class.clone()
11413            {
11414                // The chain ended here (no successor, or a repeated failure).
11415                let h = state
11416                    .seat_history
11417                    .entry(originals[i].seat.key.clone())
11418                    .or_default();
11419                h.failed.insert(specs[i].id.clone());
11420                h.last_fail = Some(cur);
11421            }
11422            let parsed = match out {
11423                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11424                AgentOutcome::Quota(o) => {
11425                    last_quota[i] = o.quota.clone();
11426                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11427                }
11428                // Not a parseable answer: the nudge loop re-asks it, which is
11429                // exactly what a dropped stream needs. Just don't hand its raw
11430                // error JSON to `extract_json`.
11431                AgentOutcome::Dropped(o) => {
11432                    let why = o
11433                        .dropped
11434                        .as_ref()
11435                        .map(|d| d.why.as_str())
11436                        .unwrap_or("the CLI ended the stream without delivering its answer");
11437                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11438                }
11439                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11440                    Err(anyhow::anyhow!(e))
11441                }
11442            };
11443            let quota = class == Some(FailClass::Quota);
11444            let failed = parsed.is_err();
11445            if !failed && matches!(ctx.node, "judge" | "vote") {
11446                // A validated answer restores this seat's quorum presence,
11447                // including on the normal path after an all-seat block.
11448                state
11449                    .quota
11450                    .retain(|loss| loss.seat != originals[i].seat.key);
11451            }
11452            done[i] = Some(parsed);
11453            if carry && !failed {
11454                let h = state
11455                    .seat_history
11456                    .entry(originals[i].seat.key.clone())
11457                    .or_default();
11458                h.failed.remove(&specs[i].id);
11459                h.last_ok = Some(specs[i].id.clone());
11460                h.last_fail = None;
11461            }
11462            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11463            // fail the same way; and never re-ask a seat that already parsed.
11464            // A failed agent that still has a successor is not re-asked
11465            // either: the handover was its remedy and has just been refused
11466            // (the chain stops on a repeated failure class). A seat with no
11467            // successor left (a single-agent roster, the roster's tail, or an
11468            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11469            let agent_failure = class.is_some()
11470                && !nudge_first
11471                && !roster.is_empty()
11472                && next_agent(i, &tried[i], &specs).is_some();
11473            if failed && !quota && !agent_failure && nudges[i] < retries {
11474                nudges[i] += 1;
11475                still.push(i);
11476            }
11477        }
11478        pending = still;
11479    }
11480
11481    // One loss per seat whose chain ended on a quota: the intermediate ones
11482    // were absorbed by a handover and are not losses.
11483    for (i, q) in last_quota.into_iter().enumerate() {
11484        if let Some(quota) = q {
11485            losses.push(QuotaLoss::from_report(
11486                originals[i].seat.key.clone(),
11487                ctx.node,
11488                Some(&quota),
11489            ));
11490        }
11491    }
11492
11493    // A stalled quorum already has its own recovery checkpoint. Refresh
11494    // its missing seats before deciding whether it can resume.
11495    if state.status != RunStatus::Stalled
11496        && ctx.node != "deliberate"
11497        && n > 0
11498        && losses.len() == n
11499        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11500    {
11501        block_machine_outage(state, ctx.node, losses);
11502        state.quota.append(losses);
11503    }
11504
11505    seats
11506        .into_iter()
11507        .zip(done)
11508        .zip(nudges)
11509        .map(|((seat, res), attempts)| {
11510            (
11511                seat,
11512                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11513                attempts,
11514            )
11515        })
11516        .collect()
11517}
11518
11519/// A node with no available agent has no task verdict. Preserve the node
11520/// checkpoint and record why a subsequent explicit resume is needed.
11521pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11522    let mut auth: Vec<_> = losses
11523        .iter()
11524        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11525        .map(|loss| loss.agent.clone())
11526        .collect();
11527    auth.sort();
11528    auth.dedup();
11529    let (kind, detail) = if auth.is_empty() {
11530        (
11531            "agent usage limit",
11532            "All seats reached their agent usage limits; resume after usage is available"
11533                .to_owned(),
11534        )
11535    } else {
11536        (
11537            "agent authentication",
11538            format!(
11539                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11540                auth.join(", ")
11541            ),
11542        )
11543    };
11544    state.resource_block = Some(crate::run::ResourceBlock {
11545        node: node.to_owned(),
11546        kind: kind.to_owned(),
11547        cause: crate::run::ResourceBlockCause::Other,
11548        detail: detail.clone(),
11549        at: Timestamp::now(),
11550    });
11551    state.event(node, detail);
11552}
11553
11554/// Did the wave end without asking any agent (see [`block_wave`])?
11555fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11556    results
11557        .iter()
11558        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11559}
11560
11561/// Settle a wave that never spawned anything: every job answers
11562/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11563/// else about the seat (session, history, quota) is touched.
11564fn block_wave(
11565    jobs: Vec<SeatJob>,
11566    state: &mut RunState,
11567    node: &str,
11568    attempt: usize,
11569    why: &str,
11570) -> Vec<(usize, SeatState, AgentOutcome)> {
11571    state.resource_wait = None;
11572    state
11573        .active
11574        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11575    if let Err(e) = state.save() {
11576        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11577    }
11578    jobs.into_iter()
11579        .enumerate()
11580        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11581        .collect()
11582}
11583
11584/// Acquire the shared build cache's lease, waiting out contention within
11585/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11586/// unbounded wait is never acceptable).
11587///
11588/// A first, non-blocking check happens before ever waiting; if it finds the
11589/// lease busy, that fact is logged as a `verify` event *and* flushed with
11590/// [`RunState::save`] immediately — not only once the wait finally succeeds
11591/// or gives up — so a `magi show` run by a different process while this one
11592/// is still waiting reads a `run.json` that says so, rather than whatever it
11593/// looked like before the wait started. The same applies to the terminal
11594/// failure: logged and saved before this returns `Err`, so a caller that
11595/// could not get the lease at all still leaves a legible record of why.
11596async fn acquire_cache_lease(
11597    state: &mut RunState,
11598    cache_dir: &Path,
11599    owner: &crate::cache::Owner,
11600    budget: Duration,
11601    node: &str,
11602    context: &str,
11603    cancel: &(dyn Fn() -> bool + Sync),
11604) -> Result<Option<crate::cache::Guard>> {
11605    let home = crate::run::home();
11606    let started = Instant::now();
11607    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11608        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11609        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11610        Err(e) => {
11611            state.event(
11612                "verify",
11613                format!("{context}: could not check the shared build cache: {e:#}"),
11614            );
11615            if let Err(e2) = state.save() {
11616                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11617            }
11618            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11619        }
11620    };
11621    state.event(
11622        "verify",
11623        format!(
11624            "{context}: waiting for the shared build cache at {} ({})",
11625            cache_dir.display(),
11626            busy.describe()
11627        ),
11628    );
11629    // Seats of this node are waiting, not running: say so, so the countdown a
11630    // reader sees only ever measures an agent that has been spawned.
11631    state.resource_wait = Some(crate::run::ResourceWait {
11632        node: context.to_owned(),
11633        kind: "shared build cache".to_owned(),
11634        holder: busy.describe(),
11635        since: Timestamp::now(),
11636        limit_secs: budget.as_secs(),
11637    });
11638    if let Err(e) = state.save() {
11639        tracing::warn!("could not persist a cache wait: {e:#}");
11640    }
11641    let remaining = budget.saturating_sub(started.elapsed());
11642    let waited = crate::cache::wait_for_unless(
11643        &home,
11644        cache_dir,
11645        owner,
11646        remaining,
11647        Duration::from_secs(5),
11648        cancel,
11649    )
11650    .await;
11651    state.resource_wait = None;
11652    let cause = match &waited {
11653        Ok(Some(_)) => None,
11654        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11655        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11656            Some(crate::run::ResourceBlockCause::Timeout {
11657                waited_secs: started.elapsed().as_secs(),
11658                limit_secs: budget.as_secs(),
11659            })
11660        }
11661        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11662    };
11663    if let Some(cause) = cause {
11664        let detail = match &waited {
11665            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11666            Err(error) => format!("{error:#}"),
11667        };
11668        state.resource_block = Some(crate::run::ResourceBlock {
11669            cause,
11670            node: node.to_owned(),
11671            kind: "shared build cache".into(),
11672            detail,
11673            at: Timestamp::now(),
11674        });
11675        state.event(
11676            "resource",
11677            state.resource_block.as_ref().unwrap().description(),
11678        );
11679    }
11680    if let Err(error) = state.save() {
11681        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11682    }
11683    waited
11684}
11685
11686/// The lease state could not be read at all (unreadable lease file, a
11687/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11688/// known to be contended, so a wave degrades to running without a shared cache
11689/// instead of stopping on a hold that [`crate::triage`] could never release.
11690#[derive(Debug)]
11691struct CacheCheckError(String);
11692
11693impl std::fmt::Display for CacheCheckError {
11694    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11695        f.write_str(&self.0)
11696    }
11697}
11698
11699impl std::error::Error for CacheCheckError {}
11700
11701/// Keep a specific acquisition outcome when the caller adds command evidence.
11702fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11703    state
11704        .resource_block
11705        .get_or_insert_with(|| crate::run::ResourceBlock {
11706            cause: crate::run::ResourceBlockCause::Other,
11707            node: node.to_owned(),
11708            kind: "shared build cache".into(),
11709            detail: format!("{error:#}"),
11710            at: Timestamp::now(),
11711        });
11712    if let Err(error) = state.save() {
11713        tracing::warn!("could not persist cache resource block: {error:#}");
11714    }
11715}
11716
11717/// Run `body` — a verify command batch — while holding the shared build
11718/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11719/// never interleave with another borrower's build against the same
11720/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11721/// timeout, or a human's own `magi review`. See the `cache` module doc for
11722/// why this matters more than Cargo's own per-target locking covers — two
11723/// *different* worktrees building the same package name/version into one
11724/// cache directory is a staleness bug, not a lock contention one.
11725///
11726/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11727/// agent waves do. Repository scheduling also has its own bounded,
11728/// cancellable resource-wait policy. Contention must not consume the
11729/// verification budget: `body` receives the full `budget` after both leases
11730/// are acquired and the cache is fresh. Without a shared cache, only
11731/// repository coordination is acquired before handing over the full budget.
11732///
11733/// A lease that cannot be acquired within the resource wait is reported as a single
11734/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11735/// verification — the same shape a spawn failure already takes in
11736/// [`run_commands`], so a caller need not special-case it.
11737#[allow(clippy::too_many_arguments)]
11738async fn with_cache_lease<'s, F, Fut>(
11739    state: &'s mut RunState,
11740    cache_dir: Option<&Path>,
11741    node: &str,
11742    seat: &str,
11743    worktree: &Path,
11744    head: &str,
11745    budget: Duration,
11746    context: &str,
11747    wait: &crate::repo_lock::WaitPolicy,
11748    body: F,
11749) -> Result<(Vec<CommandOutcome>, bool)>
11750where
11751    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11752    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11753{
11754    // Match editing waves: cache first, repository second. In particular,
11755    // pre_gate must never hold the repository while waiting for a wave's cache.
11756    let started = Instant::now();
11757    let home = crate::run::home();
11758    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11759    let guard = if let Some(cache_dir) = cache_dir {
11760        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11761        match acquire_cache_lease(
11762            state,
11763            cache_dir,
11764            &owner,
11765            resource_limit,
11766            node,
11767            context,
11768            &|| wait.cancelled(),
11769        )
11770        .await
11771        .and_then(|g| {
11772            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11773        }) {
11774            Ok(g) => Some(g),
11775            Err(e) => {
11776                record_cache_error(state, node, &e);
11777                return Ok((
11778                    vec![CommandOutcome {
11779                        command: "(waiting for the shared build cache)".into(),
11780                        code: None,
11781                        output_tail: e.to_string(),
11782                        duration_ms: started.elapsed().as_millis() as u64,
11783                        resource_blocked: true,
11784                    }],
11785                    false,
11786                ));
11787            }
11788        }
11789    } else {
11790        None
11791    };
11792    let _coordination = coordinate_operation(state, worktree, wait).await?;
11793    if let Some(cache_dir) = cache_dir {
11794        let identity = crate::cache::Identity::new(worktree, head);
11795        if let Err(e) = crate::cache::ensure_fresh(
11796            &home,
11797            cache_dir,
11798            &identity,
11799            &state.config.verify.cargo_manifests(),
11800        ) {
11801            record_cache_error(state, node, &e);
11802            state.event(
11803                "verify",
11804                format!(
11805                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11806                    worktree.display(),
11807                    short(head)
11808                ),
11809            );
11810            if let Some(guard) = guard {
11811                guard.release();
11812            }
11813            return Ok((
11814                vec![CommandOutcome {
11815                    command: "(confirming the shared build cache is fresh)".into(),
11816                    code: None,
11817                    output_tail: e.to_string(),
11818                    duration_ms: started.elapsed().as_millis() as u64,
11819                    resource_blocked: true,
11820                }],
11821                false,
11822            ));
11823        }
11824    }
11825    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11826    if !timed_out_pids.is_empty() {
11827        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11828    }
11829    // Drop repository coordination before releasing the cache, reversing the
11830    // acquisition order. A cancelled/error future also drops both guards.
11831    drop(_coordination);
11832    if let Some(guard) = guard {
11833        guard.release();
11834    }
11835    Ok((outcomes, retried))
11836}
11837
11838/// Poll `pids` — commands [`run_commands`] reports as still running when its
11839/// own timeout elapsed — until every one is confirmed gone, or
11840/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11841///
11842/// Real confirmation where confirmation is possible, not a substitute for
11843/// full process-tree observation: a grandchild the timed-out process spawned
11844/// and that survives independently of it is invisible to a pid check the
11845/// same way it always was, and continuing to observe and collect *that*
11846/// stays a different piece of work with its own owner. This only narrows a
11847/// fixed blind wait into an actual check of the pids this process does know
11848/// about.
11849async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11850    wait_for_pids_with(
11851        pids,
11852        crate::proc::pid_alive,
11853        LEASE_RELEASE_POLL,
11854        LEASE_RELEASE_MAX_WAIT,
11855    )
11856    .await;
11857}
11858
11859/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11860/// interval and ceiling supplied by the caller, so the polling *logic* -
11861/// returns as soon as every pid reports dead, gives up at the ceiling
11862/// otherwise - is testable on millisecond durations without asking the real
11863/// OS about a pid at all.
11864async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11865    pids: &[u32],
11866    alive: F,
11867    poll: Duration,
11868    max_wait: Duration,
11869) {
11870    let deadline = Instant::now() + max_wait;
11871    loop {
11872        if pids.iter().all(|&pid| !alive(pid)) {
11873            return;
11874        }
11875        if Instant::now() >= deadline {
11876            return;
11877        }
11878        tokio::time::sleep(poll).await;
11879    }
11880}
11881
11882/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11883/// admission that it could not even get a verify command to run, as opposed
11884/// to evidence the command actually produced? A caller that would otherwise
11885/// read a resource-blocked outcome as a red command must check this first:
11886/// see [`Runner::gate`], which retries rather than records `Blocked` when
11887/// this is true.
11888fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11889    outcomes.iter().any(|o| o.resource_blocked)
11890}
11891
11892/// What [`Runner::gate_fix_round`] decided.
11893pub(crate) enum GateFix {
11894    /// The tree changed and `verify.e2e` is still green: run the gate again.
11895    Retry,
11896    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11897    /// last failure stands and the run ends blocked.
11898    Stop,
11899    /// `verify.e2e` could not run after the fix (magi's own contention):
11900    /// decide nothing now, a later reentry retries.
11901    Defer,
11902}
11903
11904/// Is every red command in `outcomes` an ordinary failure the code could
11905/// explain: it ran, exited non-zero, and said something?
11906///
11907/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11908/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11909/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11910/// what the command is stays the gate's business.
11911pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11912    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11913    red.peek().is_some()
11914        && red.all(|o| {
11915            !o.resource_blocked
11916                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11917                && !o.output_tail.trim().is_empty()
11918        })
11919}
11920
11921/// Describe one verify command's outcome for the event log, distinguishing a
11922/// build/link failure — the toolchain never produced a binary to run — from
11923/// an actual test failure, since only the latter is a verdict on the patch.
11924fn e2e_outcome_label(o: &CommandOutcome) -> String {
11925    if o.ok() {
11926        return "pass".to_owned();
11927    }
11928    let reason = if o.build_failed() {
11929        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11930    } else {
11931        format!("FAIL ({:?})", o.code)
11932    };
11933    format!(
11934        "{reason}\n{}",
11935        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11936    )
11937}
11938
11939/// Run `verify.e2e`, retrying once if the first attempt could not build or
11940/// link — a build/link failure is frequently a race against a shared
11941/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11942/// `verify` event per command, tagged with `context` (normally `"round N"`)
11943/// so the two call sites that need this — the ordinary per-round leg in
11944/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11945/// it will ever call a round green — read identically in the event log.
11946async fn run_e2e_with_retry(
11947    state: &mut RunState,
11948    shell: &[String],
11949    commands: &[String],
11950    worktree: &Path,
11951    timeout: Duration,
11952    context: &str,
11953) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11954    let (mut e2e, mut timed_out_pids) = run_commands(
11955        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11956    )
11957    .await;
11958    for o in &e2e {
11959        state.event(
11960            "verify",
11961            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11962        );
11963    }
11964    // A build/link failure is not a verdict on the patch — it is frequently a
11965    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11966    // one retry before letting a red like that decide the round.
11967    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11968    if verify_retried {
11969        state.event(
11970            "verify",
11971            format!(
11972                "{context}: verify could not build/link, not a test result — retrying once \
11973                 before concluding"
11974            ),
11975        );
11976        let retried = run_commands(
11977            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11978        )
11979        .await;
11980        e2e = retried.0;
11981        // Both attempts' timeouts matter, not just the last one: the first
11982        // attempt's descendants may still be alive alongside the retry's.
11983        timed_out_pids.extend(retried.1);
11984        for o in &e2e {
11985            state.event(
11986                "verify",
11987                format!(
11988                    "{context}: retry `{}` -> {}",
11989                    o.command,
11990                    e2e_outcome_label(o)
11991                ),
11992            );
11993        }
11994    }
11995    (e2e, verify_retried, timed_out_pids)
11996}
11997
11998/// Run configured shell commands in `cwd`, in order. The second element is
11999/// the pid of every command that hit `timeout` and was still running when
12000/// this stopped waiting on it (best-effort: `None` when the platform did not
12001/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12002/// that releases a shared resource afterward needs to know.
12003///
12004/// Records `task` into [`RunState::active`] at every command boundary
12005/// (`RunState::task_command`) and clears it once the whole list has run
12006/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12007/// for minutes with no seat and no output of its own to show for it (see
12008/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12009/// apart from "not yet run" without this), and this is the only place that
12010/// knows which command is running right now and how many are left. Three
12011/// saves per command — start, not per second — matching the same "only at a
12012/// boundary" rule [`wave`] already follows for seats.
12013#[allow(clippy::too_many_arguments)]
12014async fn run_commands(
12015    state: &mut RunState,
12016    node: &str,
12017    task: &str,
12018    attempt: usize,
12019    shell: &[String],
12020    commands: &[String],
12021    cwd: &Path,
12022    timeout: Duration,
12023) -> (Vec<CommandOutcome>, Vec<u32>) {
12024    if commands.is_empty() {
12025        // Nothing to mark as running and nothing to clear — an empty list
12026        // means "not configured", and touching `active` (or the disk) over
12027        // that would be a write for every round of a repo with no
12028        // `verify.e2e` / `verify.gate` commands at all.
12029        return (Vec::new(), Vec::new());
12030    }
12031    let mut out = Vec::new();
12032    let mut timed_out_pids = Vec::new();
12033    let total = commands.len();
12034    for (idx, command) in commands.iter().enumerate() {
12035        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12036        if let Err(e) = state.save() {
12037            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12038        }
12039        let started = Instant::now();
12040        let mut cmd = tokio::process::Command::new(&shell[0]);
12041        cmd.quiet();
12042        // Verify the checkout as a standalone caller. Worker ownership and
12043        // pinned executable selection belong to this driver, not to tests or
12044        // tools it runs. Preserve home/seat context and Running's containment.
12045        for key in crate::worker::INVOCATION_ENV {
12046            cmd.env_remove(key);
12047        }
12048        cmd.args(&shell[1..])
12049            .arg(command)
12050            .current_dir(cwd)
12051            .stdin(std::process::Stdio::null())
12052            .stdout(std::process::Stdio::piped())
12053            .stderr(std::process::Stdio::piped())
12054            .kill_on_drop(true);
12055        let mut outputs = Vec::new();
12056        let protected = (|| -> Result<()> {
12057            crate::incremental::configure(&mut cmd, cwd)?;
12058            let mut paths = vec![cwd.to_owned()];
12059            if let Some(cache) = state.config.cache_dir() {
12060                paths.push(cache);
12061            }
12062            outputs.extend(crate::retention::OutputLease::admit(
12063                &crate::run::home(),
12064                &paths,
12065                state.config.disk.min_free_bytes,
12066            )?);
12067            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12068                outputs.push(guard);
12069            }
12070            if let Some(cache) = state.config.cache_dir() {
12071                outputs.push(crate::retention::OutputLease::cache(
12072                    &crate::run::home(),
12073                    &cache,
12074                    false,
12075                )?);
12076            }
12077            Ok(())
12078        })();
12079        if let Err(e) = protected {
12080            let detail = format!("execution admission unavailable: {e:#}");
12081            state.resource_block = Some(crate::run::ResourceBlock {
12082                cause: crate::run::ResourceBlockCause::Other,
12083                node: node.to_owned(),
12084                kind: if detail.contains("disk admission") {
12085                    "disk admission"
12086                } else {
12087                    "output ownership"
12088                }
12089                .to_owned(),
12090                detail: detail.clone(),
12091                at: Timestamp::now(),
12092            });
12093            out.push(CommandOutcome {
12094                command: command.clone(),
12095                code: None,
12096                output_tail: detail,
12097                duration_ms: 0,
12098                resource_blocked: true,
12099            });
12100            break;
12101        }
12102        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12103        let (code, body) = match spawned {
12104            Ok(mut running) => {
12105                let pid = running.child().id();
12106                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12107                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12108                let mut paths = vec![cwd.to_owned()];
12109                if let Some(cache) = state.config.cache_dir() {
12110                    paths.push(cache);
12111                }
12112                let mut pressure = None;
12113                let status = tokio::select! {
12114                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12115                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12116                        pressure = Some(reason); None
12117                    }
12118                };
12119                let (code, failure) = match status {
12120                    Some(Ok(Ok(status))) => (
12121                        status.code(),
12122                        status
12123                            .code()
12124                            .is_none()
12125                            .then(|| format!("command terminated: {status}")),
12126                    ),
12127                    Some(Ok(Err(error))) => {
12128                        (None, Some(format!("failed waiting for command: {error}")))
12129                    }
12130                    Some(Err(_)) => {
12131                        // Leased cleanup confirms process-tree death itself.
12132                        // Unleased commands still need the caller's PID check
12133                        // after kill-on-drop, before releasing cache ownership.
12134                        if !running.coordinated()
12135                            && let Some(pid) = pid
12136                        {
12137                            timed_out_pids.push(pid);
12138                        }
12139                        (
12140                            None,
12141                            Some(format!("timed out after {}s", timeout.as_secs())),
12142                        )
12143                    }
12144                    None => (None, None),
12145                };
12146                running.finish().await;
12147                if let Some(reason) = pressure {
12148                    state.resource_block = Some(crate::run::ResourceBlock {
12149                        cause: crate::run::ResourceBlockCause::Other,
12150                        node: node.to_owned(),
12151                        kind: "disk pressure".to_owned(),
12152                        detail: reason.clone(),
12153                        at: Timestamp::now(),
12154                    });
12155                    out.push(CommandOutcome {
12156                        command: command.clone(),
12157                        code: None,
12158                        output_tail: reason,
12159                        duration_ms: started.elapsed().as_millis() as u64,
12160                        resource_blocked: true,
12161                    });
12162                    break;
12163                }
12164                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12165                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12166                if let Some(failure) = failure {
12167                    body.push('\n');
12168                    body.push_str(&failure);
12169                }
12170                (code, body)
12171            }
12172            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12173        };
12174        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12175        if !body.is_empty() {
12176            // The full text is kept whatever the summary dropped. A fresh
12177            // name per command so a resume or retry never overwrites an
12178            // earlier run's output. A failed write changes no verdict.
12179            match crate::run::write_verification_artifact(state, task, &body) {
12180                Ok(name) if summary != body => {
12181                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12182                }
12183                Ok(_) => {}
12184                Err(e) => {
12185                    let warning = format!("could not save the full {task} output: {e:#}");
12186                    tracing::warn!("{warning}");
12187                    state.event(node, warning);
12188                    if summary != body {
12189                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12190                    }
12191                }
12192            }
12193        }
12194        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12195        out.push(CommandOutcome {
12196            command: command.clone(),
12197            code,
12198            output_tail: summary,
12199            duration_ms: started.elapsed().as_millis() as u64,
12200            resource_blocked,
12201        });
12202        if resource_blocked {
12203            state.resource_block = Some(crate::run::ResourceBlock {
12204                cause: crate::run::ResourceBlockCause::Other,
12205                node: node.to_owned(),
12206                kind: "incremental cache admission".into(),
12207                detail: body,
12208                at: Timestamp::now(),
12209            });
12210            break;
12211        }
12212    }
12213    state.task_finished(task);
12214    if let Err(e) = state.save() {
12215        tracing::warn!("could not persist the end of {task}: {e:#}");
12216    }
12217    (out, timed_out_pids)
12218}
12219
12220/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12221/// section (`report::run`) and in the `merge` event this node records — for
12222/// the operator to run by hand.
12223///
12224/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12225/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12226/// merge commits", or "require linear history") rejects the push a `--no-ff`
12227/// merge would produce, which is exactly the guidance this function replaces.
12228/// `message`'s first line becomes the squash commit's subject, matching the
12229/// note `report::run` prints alongside this command — see that function for
12230/// why an explicit subject is not optional there.
12231fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12232    let repo = repo.display();
12233    match style {
12234        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12235        MergeStyle::Squash => {
12236            // The subject sits inside double quotes, and a title an agent
12237            // wrote may carry the characters that break out of them.
12238            let subject = message
12239                .lines()
12240                .next()
12241                .unwrap_or(branch)
12242                .replace(['\\', '"', '$', '`'], "");
12243            format!(
12244                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12245            )
12246        }
12247        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12248    }
12249}
12250
12251/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12252/// under the hood, rejects a `title` over 256 characters and the whole
12253/// command fails — no PR at all, for a run whose body was otherwise fine
12254/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12255/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12256/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12257/// this bound. It is a margin, not a guarantee — a title packed
12258/// with multi-unit characters could still in principle land close to the
12259/// edge, but a real task title's occasional emoji or accented letter fits
12260/// comfortably inside it.
12261const PR_TITLE_MAX: usize = 240;
12262
12263/// A pull request title from the opening line of `text`, or `None` when that
12264/// line is not English (GitHub text is) or has no letters.
12265///
12266/// A line within `max` is kept as is. A longer one is cut at the end of its
12267/// first sentence when that falls inside `max`, else at a word boundary with a
12268/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12269/// make every merely-truncated title look non-English). The language check
12270/// runs on the kept text before any mark is added, so only what GitHub will
12271/// show is judged: an English opening followed by non-ASCII far past the cut
12272/// still passes.
12273fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12274    let line = queue::first_line(text)?;
12275    let chars: Vec<char> = line.chars().collect();
12276    let (kept, mark) = if chars.len() <= max {
12277        (line.to_owned(), "")
12278    } else if let Some(end) = sentence_end(&chars, max) {
12279        (chars[..end].iter().collect::<String>(), "")
12280    } else {
12281        let room = max.saturating_sub(3);
12282        // Cut at the last space inside the room; when the char just past the
12283        // room is a space the room already ends on a word.
12284        let cut = if chars[room].is_whitespace() {
12285            room
12286        } else {
12287            chars[..room]
12288                .iter()
12289                .rposition(|c| c.is_whitespace())
12290                .unwrap_or(room)
12291        };
12292        let head: String = chars[..cut].iter().collect();
12293        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12294        (head.to_owned(), "...")
12295    };
12296    if kept.is_empty() {
12297        return None;
12298    }
12299    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12300        return None;
12301    }
12302    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12303        return None;
12304    }
12305    Some(format!("{kept}{mark}"))
12306}
12307
12308/// The char length of the first sentence of `chars` when it ends within `max`
12309/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12310/// abbreviations so `e.g. foo` does not end a title early.
12311fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12312    const MIN: usize = 20;
12313    for i in MIN..max.min(chars.len()) {
12314        if !matches!(chars[i], '.' | '!' | '?') {
12315            continue;
12316        }
12317        let Some(&next) = chars.get(i + 1) else {
12318            continue;
12319        };
12320        if !next.is_whitespace() {
12321            continue;
12322        }
12323        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12324        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12325            continue;
12326        }
12327        let word: String = chars[..i]
12328            .iter()
12329            .rev()
12330            .take_while(|c| !c.is_whitespace())
12331            .collect::<Vec<_>>()
12332            .into_iter()
12333            .rev()
12334            .collect();
12335        let word = word.to_ascii_lowercase();
12336        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12337            continue;
12338        }
12339        let end = chars[..i]
12340            .iter()
12341            .rposition(|c| !c.is_whitespace())
12342            .map_or(i, |p| p + 1);
12343        return Some(end);
12344    }
12345    None
12346}
12347
12348/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12349/// change: a title and a body describing what was *implemented*, not the task
12350/// that asked for it. A task reads as a request; a reader of the merged
12351/// history wants the change.
12352struct PrMessage {
12353    title: String,
12354    body: String,
12355}
12356
12357impl PrMessage {
12358    /// Title, blank line, body. The first line is the squash/merge commit
12359    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12360    /// has to stay one sensible line.
12361    fn commit_message(&self) -> String {
12362        format!("{}\n\n{}", self.title, self.body)
12363    }
12364}
12365
12366/// The text after a leading `TITLE:` (any case) on `line`.
12367fn title_marker(line: &str) -> Option<&str> {
12368    let line = line.trim();
12369    let head = line.get(..6)?;
12370    head.eq_ignore_ascii_case("title:")
12371        .then(|| line[6..].trim())
12372}
12373
12374/// The implementer's own one-line title: the `TITLE:` line the implement
12375/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12376/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12377/// source, and a title that says as much is refused here too.
12378fn summary_title(summary: &str) -> Option<String> {
12379    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12380    let raw = title_marker(first)?;
12381    if raw.is_empty() {
12382        return None;
12383    }
12384    let title = queue::title_from(raw, PR_TITLE_MAX);
12385    let lower = title.to_ascii_lowercase();
12386    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12387        return None;
12388    }
12389    Some(title)
12390}
12391
12392/// How `open_review`'s instruction begins; see [`landing_title`].
12393const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12394
12395/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12396pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12397
12398/// Does this run review work that already existed, rather than implement a
12399/// task? Runs recorded before `reviewed_commits` existed carry only the
12400/// candidate marker.
12401pub(crate) fn is_review_run(state: &RunState) -> bool {
12402    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12403}
12404
12405/// The title of a review-only run: the subject of the oldest commit under
12406/// review. Later commits are usually fixups, and `instruction` is the review
12407/// prompt, which says nothing about the change. GitHub text is English, so a
12408/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12409fn review_title(state: &RunState) -> Option<String> {
12410    english_subject(
12411        state.reviewed_commits.as_ref()?.first()?,
12412        state.config.graph.github_english(),
12413    )
12414}
12415
12416/// `raw` as a pull request title, or `None` when it is blank, not English
12417/// (GitHub text is), or one of magi's own candidate commit subjects.
12418fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12419    let raw = raw.trim();
12420    if raw.is_empty() {
12421        return None;
12422    }
12423    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12424        return None;
12425    }
12426    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12427        return None;
12428    }
12429    let title = queue::title_from(raw, PR_TITLE_MAX);
12430    let lower = title.to_ascii_lowercase();
12431    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12432        return None;
12433    }
12434    Some(title)
12435}
12436
12437/// What a review-only run's branch says about itself, read at the moment the
12438/// pull request is opened.
12439#[derive(Debug, Clone, PartialEq, Eq)]
12440struct BranchFacts {
12441    /// `(subject, body)` of each commit, oldest first.
12442    commits: Vec<(String, String)>,
12443    /// Trimmed `git diff --stat`.
12444    stat: String,
12445}
12446
12447/// Longest diff stat shown: this many file lines plus the summary line.
12448const STAT_FILE_LINES: usize = 25;
12449/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12450const COMMITS_MAX_CHARS: usize = 20_000;
12451
12452/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12453/// say or finds nothing, so the caller falls back to what the run recorded.
12454async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12455    let commits = git::commit_log(repo, base, branch).await.ok()?;
12456    if commits.is_empty() {
12457        return None;
12458    }
12459    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12460    let lines: Vec<&str> = stat.lines().collect();
12461    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12462        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12463        let more = format!(" ... {omitted} more file(s)");
12464        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12465        kept.push(&more);
12466        kept.push(lines[lines.len() - 1]);
12467        kept.join("\n")
12468    } else {
12469        lines.join("\n")
12470    };
12471    Some(BranchFacts { commits, stat })
12472}
12473
12474/// Defang what would break the surrounding markdown: a closing `</details>`
12475/// and a code fence.
12476fn markdown_safe(text: &str) -> String {
12477    text.replace("</details>", "&lt;/details&gt;")
12478        .replace("\x60\x60\x60", "~~~")
12479}
12480
12481fn neutral_title(state: &RunState, winner: char) -> String {
12482    format!(
12483        "chore: land candidate {} of run {}",
12484        winner.to_ascii_uppercase(),
12485        state.id
12486    )
12487}
12488
12489/// Reuse the graph's bounded fixer without starting another graph or driver.
12490/// Copy the working state back even on errors so spent rounds remain visible.
12491pub(crate) async fn repair_landing_gate(
12492    state: &mut RunState,
12493    winner: &Candidate,
12494    outcomes: &[CommandOutcome],
12495) -> Result<GateFix> {
12496    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12497        return Ok(GateFix::Stop);
12498    }
12499    let mut runner = Runner {
12500        roles: state.config.resolve_roles()?,
12501        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12502        state: state.clone(),
12503        pause: Pause::new(),
12504        interrupt: Pause::new(),
12505    };
12506    let result = runner.gate_fix_round(winner, outcomes).await;
12507    *state = runner.state;
12508    result
12509}
12510
12511/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12512/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12513pub(crate) async fn verify_current_head(
12514    state: &mut RunState,
12515    winner: &Candidate,
12516    wait: &crate::repo_lock::WaitPolicy,
12517) -> Result<bool> {
12518    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12519    // Preserve the previous record before replacing it, including on crash recovery.
12520    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12521        state
12522            .verification_history
12523            .push(crate::run::VerificationEvidence {
12524                superseded_by: Some(head.clone()),
12525                archived_at: Some(Timestamp::now()),
12526                fixes_spent: Some(state.gate_fixes.len()),
12527                gate_head: None,
12528                gate_ran: false,
12529                gate: Vec::new(),
12530                e2e: state.head_e2e.take(),
12531            });
12532    }
12533    if let Some(e) = &state.head_e2e
12534        && e.head == head
12535    {
12536        if e.passed {
12537            return Ok(true);
12538        }
12539        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12540            state.status = RunStatus::Blocked;
12541            state.save()?;
12542            return Ok(false);
12543        }
12544    } else if state.reviews.last().is_some_and(|r| {
12545        r.verified_head.as_deref() == Some(head.as_str())
12546            && (r.e2e_status() == E2eStatus::Passed
12547                || (r.e2e_status() == E2eStatus::NotConfigured
12548                    && state.config.verify.e2e.is_empty()))
12549    }) || state
12550        .gate_repair_e2e
12551        .as_ref()
12552        .is_some_and(|e| e.head == head && e.passed)
12553    {
12554        return Ok(true);
12555    }
12556    if let Some(e) = &state.gate_repair_e2e
12557        && e.head == head
12558        && !e.passed
12559        && !e.outcomes.is_empty()
12560        && !verify_inconclusive(&e.outcomes)
12561    {
12562        state.status = RunStatus::Blocked;
12563        state.save()?;
12564        return Ok(false);
12565    }
12566    state.head_e2e = Some(crate::run::GateRepairE2e {
12567        head: head.clone(),
12568        passed: false,
12569        outcomes: Vec::new(),
12570    });
12571    state.save()?;
12572    let commands = state.config.verify.e2e.clone();
12573    let shell = state.config.shell();
12574    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12575    let cache_dir = state.config.cache_dir();
12576    let outcomes = if commands.is_empty() {
12577        Vec::new()
12578    } else {
12579        let (outcomes, _) = with_cache_lease(
12580            state,
12581            cache_dir.as_deref(),
12582            "e2e",
12583            "e2e",
12584            &winner.worktree,
12585            &head,
12586            timeout,
12587            "current head e2e",
12588            wait,
12589            |state, budget| async move {
12590                run_e2e_with_retry(
12591                    state,
12592                    &shell,
12593                    &commands,
12594                    &winner.worktree,
12595                    budget,
12596                    "current head e2e",
12597                )
12598                .await
12599            },
12600        )
12601        .await?;
12602        outcomes
12603    };
12604    let unresolved = verify_inconclusive(&outcomes);
12605    let passed = !unresolved
12606        && outcomes.iter().all(CommandOutcome::ok)
12607        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12608    if !outcomes.is_empty() && !unresolved {
12609        state.record_base_progress(&head);
12610    }
12611    state.head_e2e = Some(crate::run::GateRepairE2e {
12612        head: head.clone(),
12613        passed,
12614        outcomes,
12615    });
12616    if !passed && !unresolved {
12617        state.status = RunStatus::Blocked;
12618        state.event(
12619            "e2e",
12620            format!("verify.e2e failed on head {head}; nothing published"),
12621        );
12622    }
12623    state.save()?;
12624    Ok(passed)
12625}
12626
12627/// Recheck a repair's e2e requirement across failures, resource deferrals and
12628/// process interruptions. A successful result only applies to its exact SHA.
12629pub(crate) async fn check_gate_repair_e2e(
12630    state: &mut RunState,
12631    winner: &Candidate,
12632    wait: &crate::repo_lock::WaitPolicy,
12633) -> Result<GateFix> {
12634    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12635        return Ok(GateFix::Retry);
12636    }
12637    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12638    if state
12639        .gate_repair_e2e
12640        .as_ref()
12641        .is_some_and(|r| r.head == head && r.passed)
12642    {
12643        return Ok(GateFix::Retry);
12644    }
12645    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12646        head: head.clone(),
12647        passed: false,
12648        outcomes: Vec::new(),
12649    });
12650    state.save()?;
12651    let commands = state.config.verify.e2e.clone();
12652    let outcomes = if commands.is_empty() {
12653        Vec::new()
12654    } else {
12655        let shell = state.config.shell();
12656        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12657        let cache_dir = state.config.cache_dir();
12658        let (outcomes, _) = with_cache_lease(
12659            state,
12660            cache_dir.as_deref(),
12661            "e2e",
12662            "e2e",
12663            &winner.worktree,
12664            &head,
12665            timeout,
12666            "gate repair e2e",
12667            wait,
12668            |state, budget| async move {
12669                run_e2e_with_retry(
12670                    state,
12671                    &shell,
12672                    &commands,
12673                    &winner.worktree,
12674                    budget,
12675                    "gate repair e2e",
12676                )
12677                .await
12678            },
12679        )
12680        .await?;
12681        outcomes
12682    };
12683    let deferred = verify_inconclusive(&outcomes);
12684    let passed = outcomes.iter().all(CommandOutcome::ok)
12685        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12686    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12687        head,
12688        passed,
12689        outcomes,
12690    });
12691    state.save()?;
12692    if deferred {
12693        Ok(GateFix::Defer)
12694    } else if !passed {
12695        state.event(
12696            "gate",
12697            "verify.e2e failed after gate repair; nothing published",
12698        );
12699        Ok(GateFix::Stop)
12700    } else {
12701        Ok(GateFix::Retry)
12702    }
12703}
12704
12705/// Run the same configured gate for a landing head and a fresh graph.
12706pub(crate) async fn run_landing_gate(
12707    state: &mut RunState,
12708    winner: &Candidate,
12709    wait: &crate::repo_lock::WaitPolicy,
12710) -> Result<Vec<CommandOutcome>> {
12711    let shell = state.config.shell();
12712    let gate_commands = state.config.verify.gate.clone();
12713    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12714    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12715        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12716        Vec::new()
12717    } else {
12718        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12719        let cache_dir = state.config.cache_dir();
12720        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12721        let (outcomes, _) = with_cache_lease(
12722            state,
12723            cache_dir.as_deref(),
12724            "gate",
12725            "gate",
12726            &winner.worktree,
12727            &head,
12728            timeout,
12729            "final gate",
12730            wait,
12731            |state, budget| {
12732                let shell = shell.clone();
12733                let gate_commands = gate_commands.clone();
12734                let winner = winner.clone();
12735                async move {
12736                    Runner::run_pre_gate(state, &winner).await;
12737                    // Bind command evidence after mechanical fixes, before commands start.
12738                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12739                    let (outcomes, timed_out_pids) = run_commands(
12740                        state,
12741                        "gate",
12742                        "gate",
12743                        0,
12744                        &shell,
12745                        &gate_commands,
12746                        &winner.worktree,
12747                        budget,
12748                    )
12749                    .await;
12750                    (outcomes, false, timed_out_pids)
12751                }
12752            },
12753        )
12754        .await?;
12755        outcomes
12756    };
12757    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12758    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12759        let mut outcomes = outcomes;
12760        outcomes.push(CommandOutcome {
12761            command: "gate head stability".into(), code: None,
12762            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12763            duration_ms: 0, resource_blocked: false,
12764        });
12765        return Ok(outcomes);
12766    }
12767    if outcomes.is_empty() {
12768        // Nothing configured to check — distinct from every other
12769        // silence in this run's event log, since an empty `gate` alone
12770        // no longer says whether the gate ran at all (see
12771        // `RunState::gate_ran`'s own doc).
12772        state.event("gate", "no gate commands configured; not tested");
12773    }
12774    for o in &outcomes {
12775        state.event(
12776            "gate",
12777            format!(
12778                "`{}` -> {}",
12779                o.command,
12780                if o.ok() {
12781                    "pass".to_owned()
12782                } else {
12783                    format!(
12784                        "FAIL ({:?})\n{}",
12785                        o.code,
12786                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12787                    )
12788                }
12789            ),
12790        );
12791    }
12792    Ok(outcomes)
12793}
12794
12795/// The pull request title to hand to `land::merge_subject`. A review-only run
12796/// opened by an earlier build titled its pull request with the review prompt;
12797/// that title is dropped (empty, so the fallback applies) rather than landed.
12798/// Any other title, including an operator's rename, passes through untouched,
12799/// and so does every title of a run that implements a task.
12800pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12801    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12802        ""
12803    } else {
12804        pr_title
12805    }
12806}
12807
12808/// What the squash subject falls back to when the pull request title is empty
12809/// or candidate-shaped: for a review-only run the derived title, never the
12810/// review prompt held in `instruction`.
12811pub fn landing_subject_source(state: &RunState) -> String {
12812    if is_review_run(state) {
12813        let winner = state.candidates.first().map_or('A', |c| c.label);
12814        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12815    }
12816    state.instruction.clone()
12817}
12818
12819/// `summary` without its `TITLE:` line, which the pull request title already
12820/// carries.
12821fn summary_without_title(summary: &str) -> String {
12822    let mut lines = summary.trim().lines().peekable();
12823    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12824        lines.next();
12825    }
12826    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12827}
12828
12829/// The pull request title and body for the winning candidate.
12830///
12831/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12832/// the task's own opening line via [`queue::title_from`] when there is none.
12833/// `state.instruction` can open with blank lines (`task_text` only rejects a
12834/// body that is blank *entirely*), which `title_from` skips.
12835///
12836/// Publishing uses an independent description generated from the final diff
12837/// and structured evidence. Internal summaries and reports remain local.
12838/// Non-publishing previews may display the original task and candidate summary.
12839/// The footer retains the run and candidate provenance.
12840#[cfg(test)]
12841fn pr_message(state: &RunState, winner: char) -> PrMessage {
12842    pr_message_with(state, winner, None)
12843}
12844
12845/// [`pr_message`] with what the branch of a review-only run says about itself.
12846/// `facts` is ignored for a run that implements a task.
12847#[cfg(test)]
12848fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12849    let raw = pr_message_raw(
12850        state,
12851        winner,
12852        facts,
12853        crate::pr_outcome::FinalHead::Unreported,
12854        ChangeNote::NotAsked,
12855    );
12856    let id = crate::scrub::Identity::current();
12857    PrMessage {
12858        title: crate::scrub::scrub(&raw.title, &id),
12859        body: crate::scrub::scrub(&raw.body, &id),
12860    }
12861}
12862
12863/// What the public description may say about the change itself.
12864#[derive(Debug, Clone, Copy)]
12865enum ChangeNote<'a> {
12866    /// A local preview or non-publishing merge mode: no public generation needed.
12867    NotAsked,
12868    /// A summary written from the final diff.
12869    Summary(&'a crate::run::ChangeSummary),
12870    /// A summary was asked for and could not be produced; the text says why.
12871    Unavailable(&'a str),
12872}
12873
12874fn pr_message_raw(
12875    state: &RunState,
12876    winner: char,
12877    facts: Option<&BranchFacts>,
12878    head: crate::pr_outcome::FinalHead<'_>,
12879    change: ChangeNote<'_>,
12880) -> PrMessage {
12881    if !matches!(change, ChangeNote::NotAsked) {
12882        let (title, mut body) = match change {
12883            ChangeNote::Summary(c) => (
12884                c.title.clone(),
12885                format!("## Summary\n\n{}\n\n", c.body.trim()),
12886            ),
12887            ChangeNote::Unavailable(why) => (
12888                neutral_title(state, winner),
12889                format!(
12890                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12891                ),
12892            ),
12893            ChangeNote::NotAsked => unreachable!(),
12894        };
12895        body.push_str(&public_pr_evidence(state, head));
12896        body.push_str(&format!(
12897            "\n---\nmagi:run/{} magi:candidate-{}\n",
12898            state.id,
12899            winner.to_ascii_lowercase()
12900        ));
12901        return PrMessage { title, body };
12902    }
12903    pr_preview(state, winner, facts, head)
12904}
12905
12906/// Append public facts from the run records after every prose generation,
12907/// including posting-gate rewrites. The agent cannot drop this provenance.
12908fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12909    let mut body = crate::pr_outcome::review_rounds(state);
12910    body.push_str(&crate::pr_outcome::final_verification(state, head));
12911    let open = state.last_round_findings();
12912    if !open.is_empty() {
12913        body.push_str("## Open review findings\n\n");
12914        for f in open {
12915            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12916        }
12917        body.push('\n');
12918    }
12919    if let Some(c) = state.change_summary.as_ref()
12920        && head == crate::pr_outcome::FinalHead::At(&c.head)
12921    {
12922        body.push_str(&format!(
12923            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12924            c.base, c.head
12925        ));
12926    }
12927    body
12928}
12929
12930/// Local-only preview of the operator's task and candidate summary.
12931fn pr_preview(
12932    state: &RunState,
12933    winner: char,
12934    facts: Option<&BranchFacts>,
12935    head: crate::pr_outcome::FinalHead<'_>,
12936) -> PrMessage {
12937    let summary = state
12938        .candidates
12939        .iter()
12940        .find(|c| c.label == winner)
12941        .map(|c| c.summary.as_str())
12942        .unwrap_or_default();
12943    // The fallback is the operator's own words and may not be English; GitHub
12944    // text always is, so a non-English task gets a neutral title instead.
12945    let review = is_review_run(state);
12946    let title = if review {
12947        facts
12948            .and_then(|f| {
12949                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12950            })
12951            .or_else(|| review_title(state))
12952            .or_else(|| {
12953                state
12954                    .candidates
12955                    .iter()
12956                    .find(|c| c.label == winner)
12957                    .filter(|c| !c.branch.starts_with("magi/"))
12958                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12959            })
12960            .unwrap_or_else(|| neutral_title(state, winner))
12961    } else {
12962        summary_title(summary).unwrap_or_else(|| {
12963            english_title(
12964                &state.instruction,
12965                PR_TITLE_MAX,
12966                state.config.graph.github_english(),
12967            )
12968            .unwrap_or_else(|| neutral_title(state, winner))
12969        })
12970    };
12971
12972    let mut body = String::new();
12973    let what = summary_without_title(summary);
12974    if !what.is_empty() {
12975        body.push_str("## Summary\n\n");
12976        body.push_str(&what);
12977        body.push_str("\n\n");
12978    }
12979
12980    // Review history uses structured facts across all rounds, never reports.
12981    body.push_str(&crate::pr_outcome::review_rounds(state));
12982    body.push_str(&crate::pr_outcome::final_verification(state, head));
12983    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12984
12985    let open = state.open_findings();
12986    if !open.is_empty() {
12987        body.push_str("## Open review findings\n\n");
12988        for f in &open {
12989            body.push_str(&format!(
12990                "- `{}` [{:?}] {}\n",
12991                f.id, f.severity, "Context retained in local review records."
12992            ));
12993        }
12994        body.push('\n');
12995    }
12996
12997    if let Some(fix) = fix
12998        && !fix.rejected.is_empty()
12999    {
13000        body.push_str("## Declined by the fixer\n\n");
13001        for r in &fix.rejected {
13002            body.push_str(&format!(
13003                "- `{}`: {}\n",
13004                r.id, "Reason retained in local review records."
13005            ));
13006        }
13007        body.push('\n');
13008    }
13009
13010    if review {
13011        // The review prompt is not the task; list what the branch carries.
13012        body.push_str("## Commits under review\n\n");
13013        if let Some(facts) = facts {
13014            let mut left = COMMITS_MAX_CHARS;
13015            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13016                let mut entry = format!("- {}\n", markdown_safe(subject));
13017                for l in markdown_safe(text).lines() {
13018                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13019                }
13020                if left == 0 {
13021                    body.push_str(&format!(
13022                        "- ... {} more commit(s)\n",
13023                        facts.commits.len() - i
13024                    ));
13025                    break;
13026                }
13027                if entry.len() > left {
13028                    // Even the first commit is cut: one huge body must not
13029                    // push the whole description past GitHub's limit.
13030                    let mut end = left;
13031                    while !entry.is_char_boundary(end) {
13032                        end -= 1;
13033                    }
13034                    entry.truncate(end);
13035                    entry.push_str("\n  ... (truncated)\n");
13036                    left = 0;
13037                } else {
13038                    left -= entry.len();
13039                }
13040                body.push_str(&entry);
13041            }
13042            if !facts.stat.trim().is_empty() {
13043                body.push_str(&format!(
13044                    "\n## Diff stat\n\n```\n{}\n```\n",
13045                    markdown_safe(facts.stat.trim())
13046                ));
13047            }
13048        } else {
13049            match &state.reviewed_commits {
13050                Some(subjects) => {
13051                    for s in subjects {
13052                        body.push_str(&format!("- {}\n", s.trim()));
13053                    }
13054                }
13055                None => {
13056                    // An older run kept only the prompt, with the commit list
13057                    // after its first paragraph.
13058                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13059                    body.push_str(rest.trim());
13060                    body.push('\n');
13061                }
13062            }
13063        }
13064    } else {
13065        let task = state.instruction.trim();
13066        let task = if task.is_empty() {
13067            "(empty task)"
13068        } else {
13069            task
13070        };
13071        body.push_str(&format!(
13072            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13073            task.replace("</details>", "&lt;/details&gt;")
13074        ));
13075    }
13076
13077    body.push_str(&format!(
13078        "\n---\nmagi:run/{} magi:candidate-{}\n",
13079        state.id,
13080        winner.to_ascii_lowercase()
13081    ));
13082
13083    PrMessage { title, body }
13084}
13085
13086/// Source imports have already been prepared from structured inputs. Legacy
13087/// inferred references must never instruct a seat to reconstruct those imports.
13088fn seeded_instruction(state: &RunState) -> String {
13089    let mut inputs = Vec::new();
13090    if let Some(start) = &state.sources.start_from {
13091        inputs.push(format!(
13092            "Start from `{}` at `{}` (including ancestry).",
13093            start.token, start.sha
13094        ));
13095    }
13096    for pick in &state.sources.pick_commits {
13097        inputs.push(format!(
13098            "Pick `{}` at `{}` (single commit).",
13099            pick.token, pick.sha
13100        ));
13101    }
13102    if inputs.is_empty() {
13103        state.instruction.clone()
13104    } else {
13105        format!(
13106            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13107            state.instruction,
13108            inputs.join("\n")
13109        )
13110    }
13111}
13112
13113/// Does the winner have no commits ahead of the base it would land on?
13114/// Any failure to find out reads as "not empty": the merge then behaves as it
13115/// always did rather than refusing on a guess.
13116async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13117    let base = &state.base_branch;
13118    let mut against = base.clone();
13119    if mode == MergeMode::Pr {
13120        // A pull request lands on the remote's base, never the local branch
13121        // of the same name: if that cannot be read, "not empty" is the safe
13122        // answer.
13123        let remote = &state.config.merge.remote;
13124        let tracking = format!("{remote}/{base}");
13125        let fetched = git::fetch(repo, remote, base).await;
13126        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13127            against = tracking;
13128        } else {
13129            return false;
13130        }
13131    }
13132    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13133}
13134
13135/// Why nothing was opened for an empty winner, with what the task's own
13136/// references resolved to.
13137fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13138    let mut detail = format!(
13139        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13140         nothing to open a pull request for"
13141    );
13142    detail.push_str(
13143        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13144    );
13145    detail
13146}
13147
13148/// What the `Pr` merge does once it knows whether the branch already has an
13149/// open pull request.
13150#[derive(Debug, PartialEq, Eq)]
13151enum PrPlan {
13152    Create,
13153    Adopt { url: String, title: String },
13154    Stop(String),
13155}
13156
13157/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13158/// a failed lookup -> stop with the real reason. Never guesses.
13159fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13160    match found {
13161        Ok(land::OpenPr::None) => PrPlan::Create,
13162        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13163        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13164            "several open pull requests exist for this branch, not picking one: {}",
13165            urls.join(" ")
13166        )),
13167        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13168    }
13169}
13170
13171/// `gh pr create`, returning the PR url.
13172pub(crate) async fn gh_pr_create(
13173    cwd: &Path,
13174    base: &str,
13175    head: &str,
13176    title: &str,
13177    body: &str,
13178) -> Result<String> {
13179    let out = tokio::process::Command::new("gh")
13180        .args([
13181            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13182        ])
13183        .current_dir(cwd)
13184        .quiet()
13185        .stdin(std::process::Stdio::null())
13186        .output()
13187        .await
13188        .context("spawn gh")?;
13189    if out.status.success() {
13190        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13191    } else {
13192        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13193    }
13194}
13195
13196/// Never force deletion of uncommitted work, including unknown legacy trees.
13197async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13198    match std::fs::symlink_metadata(path) {
13199        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13200            // A previous cleanup may have removed the tree before a branch
13201            // deletion or state save failed. Unregister it if still present.
13202            git::worktree_remove_clean(repo, path).await?;
13203            return Ok(());
13204        }
13205        Err(e) => return Err(e.into()),
13206        Ok(_) => {}
13207    }
13208    if !git::is_clean(path).await? {
13209        bail!(
13210            "worktree has uncommitted work; refusing removal: {}",
13211            path.display()
13212        );
13213    }
13214    if !git::worktree_remove_clean(repo, path).await? {
13215        bail!("worktree removal refused: {}", path.display());
13216    }
13217    Ok(())
13218}
13219
13220/// Tear a run's worktrees and branches down.
13221///
13222/// `home` is where the updated `run.json` is saved (via
13223/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13224/// a housekeeping pass already has its own honest `home` handed to it, and
13225/// falling through to the global here would write back through whichever
13226/// directory some other process or test pinned into that `OnceLock` first,
13227/// not the one the caller actually resolved its `runs` and `state` from.
13228pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13229    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13230        bail!("refusing fold: {reason}");
13231    }
13232    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13233        bail!("candidate preservation incomplete; refusing cleanup");
13234    }
13235    // Serialize ordinary folding with the same starts/resumes as target-only
13236    // eviction. Otherwise a second janitor could remove the whole worktree
13237    // around a live output borrower even though target eviction was fenced.
13238    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13239    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13240    let _driver = crate::driver::maintenance(home, &state.id)?;
13241    let mut output_guards = Vec::new();
13242    for entry in std::fs::read_dir(state.worktree_root())
13243        .into_iter()
13244        .flatten()
13245        .flatten()
13246    {
13247        if entry.path().join(".git").is_file() {
13248            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13249        }
13250    }
13251    let repo = state.repo.clone();
13252    // CLI, Web and automatic cleanup all enter here, independently of
13253    // Runner::fold_losers. Protect deletion and shared config release too.
13254    let wait = repository_wait(state, None);
13255    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13256        Ok(lease) => lease,
13257        Err(e) => {
13258            state.save_under(home)?;
13259            return Err(e);
13260        }
13261    };
13262    // Confirmed merges dispose of their owned sources after durable evidence
13263    // capture. Preflight every source before removing anything.
13264    let _source_outputs = if state.status == RunStatus::Merged
13265        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13266    {
13267        verify_merged_fold_sources(state, drop_winner, home).await?
13268    } else {
13269        Vec::new()
13270    };
13271    let root = state.worktree_root();
13272    let winner = state.tally.as_ref().map(|t| t.winner);
13273    let mut removed = Vec::new();
13274
13275    for i in 0..state.candidates.len() {
13276        let c = state.candidates[i].clone();
13277        let is_winner = Some(c.label) == winner;
13278        if is_winner && !drop_winner {
13279            continue;
13280        }
13281        let existed = c.worktree.exists();
13282        // A vanished directory can still have a Git registration that keeps
13283        // its branch checked out. Remove that registration before the branch.
13284        remove_preserved_worktree(&repo, &c.worktree).await?;
13285        if existed {
13286            removed.push(c.worktree.to_string_lossy().into_owned());
13287        }
13288        // A branch handed to a later run (and its pull request) is not this
13289        // run's to delete.
13290        let handed_over = state.released_branches.contains(&c.branch);
13291        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13292            if !git::branch_delete(&repo, &c.branch).await? {
13293                bail!("candidate branch removal refused");
13294            }
13295            removed.push(c.branch.clone());
13296        }
13297        state.candidates[i].folded = true;
13298    }
13299
13300    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13301        let path = name.path();
13302        let keep = !drop_winner
13303            && winner.is_some_and(|w| {
13304                path.file_name()
13305                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13306            });
13307        if keep {
13308            continue;
13309        }
13310        remove_preserved_worktree(&repo, &path).await?;
13311        removed.push(path.to_string_lossy().into_owned());
13312    }
13313
13314    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13315    // judge worktrees, so once the loop above has cleared all of them out,
13316    // the parent is a bare directory nobody else was ever going to remove -
13317    // git only ever managed what was inside it. Left alone, one of these
13318    // accumulates per fully-folded run; the operator's own machine had 74.
13319    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13320    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13321    // so does anything a run never claimed that happens to share the bay.
13322    remove_if_empty(&root);
13323
13324    if state.enabled_worktree_config && drop_winner {
13325        // A release, not a raw disable: some sibling run in this repository
13326        // may still hold its own reference (see `git::acquire_worktree_config`),
13327        // and only the last release actually turns the setting back off.
13328        git::release_worktree_config(&repo).await.ok();
13329        state.enabled_worktree_config = false;
13330    }
13331    state.save_under(home)?;
13332    Ok(removed)
13333}
13334
13335async fn verify_merged_fold_sources(
13336    state: &RunState,
13337    drop_winner: bool,
13338    home: &Path,
13339) -> Result<Vec<std::fs::File>> {
13340    let winner = state.tally.as_ref().map(|t| t.winner);
13341    let mut heads = BTreeSet::new();
13342    for candidate in &state.candidates {
13343        if Some(candidate.label) == winner && !drop_winner {
13344            continue;
13345        }
13346        anyhow::ensure!(
13347            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13348            "candidate recovery source is outside the inventoried run root: {}",
13349            candidate.worktree.display()
13350        );
13351        match std::fs::symlink_metadata(&candidate.worktree) {
13352            Ok(_) => {}
13353            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13354            Err(error) => return Err(error.into()),
13355        }
13356        if !state.released_branches.contains(&candidate.branch)
13357            && git::branch_exists(&state.repo, &candidate.branch).await?
13358        {
13359            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13360        }
13361    }
13362    let mut cargo_guards = Vec::new();
13363    let entries = match std::fs::read_dir(state.worktree_root()) {
13364        Ok(entries) => Some(entries),
13365        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13366        Err(error) => return Err(error.into()),
13367    };
13368    for entry in entries.into_iter().flatten() {
13369        let path = entry?.path();
13370        let metadata = std::fs::symlink_metadata(&path)?;
13371        anyhow::ensure!(
13372            metadata.is_dir() && !metadata.file_type().is_symlink(),
13373            "non-directory or symlink recovery source retained: {}",
13374            path.display()
13375        );
13376        if !drop_winner
13377            && winner.is_some_and(|w| {
13378                path.file_name()
13379                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13380            })
13381        {
13382            continue;
13383        }
13384        anyhow::ensure!(
13385            path.join(".git").is_file(),
13386            "unregistered recovery source retained: {}",
13387            path.display()
13388        );
13389        anyhow::ensure!(
13390            git::git(
13391                &path,
13392                &[
13393                    "status",
13394                    "--porcelain",
13395                    "--untracked-files=all",
13396                    "--ignored",
13397                    "--",
13398                    ".",
13399                    ":(exclude)target"
13400                ]
13401            )
13402            .await?
13403            .is_empty(),
13404            "source changes or ignored recovery files retained: {}",
13405            path.display()
13406        );
13407        match std::fs::symlink_metadata(path.join("target")) {
13408            Ok(_) => {
13409                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13410            }
13411            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13412            Err(error) => return Err(error.into()),
13413        }
13414        for checkpoint in [
13415            "rebase-merge",
13416            "rebase-apply",
13417            "sequencer",
13418            "CHERRY_PICK_HEAD",
13419            "MERGE_HEAD",
13420            "REBASE_HEAD",
13421            "REVERT_HEAD",
13422            "BISECT_LOG",
13423            "AUTO_MERGE",
13424            "MERGE_MSG",
13425        ] {
13426            let location = git::git(
13427                &path,
13428                &[
13429                    "rev-parse",
13430                    "--path-format=absolute",
13431                    "--git-path",
13432                    checkpoint,
13433                ],
13434            )
13435            .await?;
13436            anyhow::ensure!(
13437                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13438                if e.kind() == std::io::ErrorKind::NotFound),
13439                "Git recovery checkpoint retained: {checkpoint}"
13440            );
13441        }
13442        heads.insert(git::rev_parse(&path, "HEAD").await?);
13443    }
13444    crate::fold_evidence::preserve(state, home, &heads).await?;
13445
13446    Ok(cargo_guards)
13447}
13448
13449/// Remove `dir` if it exists and has nothing in it.
13450///
13451/// Best-effort and silent by design: a directory that is not empty (a run
13452/// whose winner is still parked there, a stray file some other process left)
13453/// is exactly the case this must refuse, and a directory that is already gone
13454/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13455/// already refuses a non-empty directory, so the emptiness check below is
13456/// belt, not suspenders - it is what keeps this from ever attempting the
13457/// removal in the case that matters, rather than trusting `remove_dir`'s
13458/// error path to have no side effects if it ever changed.
13459fn remove_if_empty(dir: &Path) {
13460    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13461        std::fs::remove_dir(dir).ok();
13462    }
13463}
13464
13465/// Severity of the worst open finding in the last review round, for reporting.
13466pub fn worst_open(state: &RunState) -> Option<Severity> {
13467    state
13468        .reviews
13469        .last()?
13470        .reviews
13471        .iter()
13472        .flat_map(|r| r.findings.iter())
13473        .map(|f| f.severity)
13474        .max()
13475}
13476
13477/// `[worktree] setup` hides its products per worktree, which needs
13478/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13479/// worktrees, like the hook's; every place that runs setup calls this first.
13480async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13481    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13482        git::acquire_worktree_config(repo).await?;
13483        state.enabled_worktree_config = true;
13484    }
13485    Ok(())
13486}
13487
13488#[cfg(test)]
13489mod tests {
13490    #[tokio::test]
13491    async fn launcher_base_fetch_requires_remote_success() {
13492        let tmp = tempfile::tempdir().unwrap();
13493        let repo = tmp.path().join("repo");
13494        let remote = tmp.path().join("remote");
13495        std::fs::create_dir(&repo).unwrap();
13496        std::fs::create_dir(&remote).unwrap();
13497        git::git(&remote, &["init", "--bare", "-b", "main"])
13498            .await
13499            .unwrap();
13500        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13501        git::git(
13502            &repo,
13503            &[
13504                "-c",
13505                "user.name=Fixture",
13506                "-c",
13507                "user.email=fixture@example.invalid",
13508                "commit",
13509                "--allow-empty",
13510                "-m",
13511                "fixture",
13512            ],
13513        )
13514        .await
13515        .unwrap();
13516        git::git(
13517            &repo,
13518            &["remote", "add", "origin", remote.to_str().unwrap()],
13519        )
13520        .await
13521        .unwrap();
13522        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13523        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13524        assert_eq!(
13525            resolve_base(&repo, "main", "origin", None)
13526                .await
13527                .unwrap()
13528                .commit,
13529            head
13530        );
13531        #[cfg(unix)]
13532        {
13533            let policy: crate::config::Coordination = toml::from_str(
13534                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13535            ).unwrap();
13536            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13537            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13538            let held = crate::repo_lock::SeatLease::operation_with(
13539                &repo,
13540                Some(&policy),
13541                &crate::repo_lock::WaitPolicy::default(),
13542            )
13543            .await
13544            .unwrap();
13545            assert!(held.is_some());
13546            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13547            tokio::pin!(fetch);
13548            assert!(
13549                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13550                    .await
13551                    .is_err()
13552            );
13553            assert!(
13554                !repo.join(".git/FETCH_HEAD").exists(),
13555                "no fetch before admission"
13556            );
13557            drop(held);
13558            let fetched = fetch.await.unwrap();
13559            assert_eq!(fetched.commit, head);
13560            assert_eq!(fetched.remote, "origin");
13561            assert_eq!(fetched.branch, "main");
13562        }
13563        // A surviving tracking ref is not successful fetch evidence.
13564        git::git(
13565            &repo,
13566            &[
13567                "remote",
13568                "set-url",
13569                "origin",
13570                tmp.path().join("missing").to_str().unwrap(),
13571            ],
13572        )
13573        .await
13574        .unwrap();
13575        let error = resolve_base(&repo, "main", "origin", None)
13576            .await
13577            .unwrap_err()
13578            .to_string();
13579        assert!(error.contains("cannot read origin/main"));
13580        assert!(error.contains("refusing to branch off"));
13581    }
13582
13583    #[test]
13584    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13585        let leaked = vec!["chore(deps): update a crate".to_owned()];
13586        let own = "fix(daemon): apply a chosen action";
13587        assert!(should_retitle("", own, &leaked));
13588        assert!(should_retitle(
13589            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13590            own,
13591            &leaked
13592        ));
13593        assert!(should_retitle(
13594            "chore: land candidate A of run 1",
13595            own,
13596            &leaked
13597        ));
13598        assert!(should_retitle(
13599            "magi: candidate A (uncommitted work)",
13600            own,
13601            &leaked
13602        ));
13603        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13604        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13605        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13606    }
13607
13608    #[test]
13609    fn pr_merge_plan_creates_adopts_or_stops() {
13610        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13611        assert_eq!(
13612            pr_merge_plan(Ok(land::OpenPr::One {
13613                url: "u".into(),
13614                title: "t".into()
13615            })),
13616            PrPlan::Adopt {
13617                url: "u".into(),
13618                title: "t".into()
13619            }
13620        );
13621        let PrPlan::Stop(many) =
13622            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13623        else {
13624            panic!("many must stop");
13625        };
13626        assert!(many.contains('a') && many.contains('b'));
13627        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13628            panic!("a failed lookup must stop");
13629        };
13630        assert!(err.contains("bad token"));
13631    }
13632
13633    use super::*;
13634    use crate::run::GateStatus;
13635    use std::collections::BTreeMap;
13636    use std::time::Duration;
13637
13638    fn conductor() -> AgentSpec {
13639        AgentSpec {
13640            id: "conductor".to_owned(),
13641            kind: crate::config::AgentKind::Command,
13642            model: None,
13643            command: vec!["true".to_owned()],
13644            extra_args: Vec::new(),
13645            env: BTreeMap::new(),
13646            prompt_delivery: None,
13647            max_turn_secs: None,
13648        }
13649    }
13650
13651    fn spec(id: &str) -> AgentSpec {
13652        AgentSpec {
13653            id: id.to_owned(),
13654            kind: crate::config::AgentKind::Command,
13655            model: None,
13656            command: vec!["true".to_owned()],
13657            extra_args: Vec::new(),
13658            env: BTreeMap::new(),
13659            prompt_delivery: None,
13660            max_turn_secs: None,
13661        }
13662    }
13663
13664    fn ids(xs: &[&str]) -> BTreeSet<String> {
13665        xs.iter().map(|s| (*s).to_owned()).collect()
13666    }
13667
13668    #[test]
13669    fn pick_successor_skips_an_agent_another_seat_holds() {
13670        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13671        let roster = [spec("a"), spec("b"), spec("c")];
13672        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13673        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13674    }
13675
13676    #[test]
13677    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13678        // The other seat started on c but was handed to d; c is free again.
13679        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13680        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13681        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13682        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13683        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13684    }
13685
13686    #[test]
13687    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13688        let roster = [spec("a"), spec("b")];
13689        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13690        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13691        let carried = ids(&["b"]);
13692        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13693        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13694    }
13695
13696    #[test]
13697    fn pick_successor_returns_none_without_any_untried_successor() {
13698        let roster = [spec("a"), spec("b")];
13699        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13700        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13701        let carried = ids(&["a"]);
13702        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13703    }
13704
13705    #[test]
13706    fn pick_successor_rescue_avoids_another_seats_occupant() {
13707        // b is a carried failure and free; a is held by the other seat.
13708        let roster = [spec("a"), spec("b"), spec("c")];
13709        let carried = ids(&["b", "c"]);
13710        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13711        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13712    }
13713
13714    #[test]
13715    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13716        let roster = [spec("a"), spec("b"), spec("c")];
13717        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13718        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13719    }
13720
13721    #[test]
13722    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13723        let roster = [spec("a"), spec("b"), spec("c")];
13724        let failed = ids(&["a", "b", "c"]);
13725        // Rescue looks at the whole roster, once per id, then runs out.
13726        let mut tried = ids(&["b"]);
13727        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13728        assert_eq!(first.id, "a");
13729        tried.insert(first.id.clone());
13730        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13731        assert_eq!(second.id, "c");
13732        tried.insert(second.id.clone());
13733        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13734    }
13735
13736    #[test]
13737    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13738        let roster = [spec("a"), spec("b")];
13739        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13740        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13741    }
13742
13743    #[test]
13744    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13745        let roster = [spec("a"), spec("b"), spec("c")];
13746        let h = SeatHistory {
13747            failed: ids(&["a"]),
13748            last_ok: Some("b".to_owned()),
13749            last_fail: None,
13750        };
13751        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13752        // A last answerer that left the roster, or later failed, is ignored.
13753        let gone = SeatHistory {
13754            last_ok: Some("zzz".to_owned()),
13755            ..h.clone()
13756        };
13757        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13758        let failed = SeatHistory {
13759            failed: ids(&["a", "b"]),
13760            last_ok: Some("b".to_owned()),
13761            last_fail: None,
13762        };
13763        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13764    }
13765
13766    #[test]
13767    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13768        let first = SeatState::new("review-1", "alpha", 7);
13769        let next = handover_seat("review-1", "gamma", 7);
13770        assert_ne!(first.claude_session, next.claude_session);
13771    }
13772
13773    #[test]
13774    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13775        let mut state = state_with_summary("x", "y");
13776        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13777        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13778        assert_ne!(a.claude_session, b.claude_session);
13779    }
13780
13781    #[test]
13782    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13783        let roster = [spec("a"), spec("b")];
13784        let h = SeatHistory {
13785            failed: ids(&["a", "b"]),
13786            ..SeatHistory::default()
13787        };
13788        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13789        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13790        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13791    }
13792
13793    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13794    // fallback loop depends on to terminate: it must walk forward from the
13795    // seat's own position, never restart at the front of the roster, and it
13796    // must never hand back an id already tried, however many times that id
13797    // happens to appear.
13798
13799    #[test]
13800    fn failure_signature_ignores_numbers_and_paths() {
13801        assert_eq!(
13802            failure_signature("exited with Some(2) and no usable output"),
13803            failure_signature("exited with Some(137) and no usable output")
13804        );
13805        assert_eq!(
13806            failure_signature("process exited with no usable output (exit code 2)"),
13807            failure_signature("process exited with no usable output (exit code 137)")
13808        );
13809        for (old, new) in [
13810            (
13811                "exited with Some(2) and no usable output",
13812                "process exited with no usable output (exit code 2)",
13813            ),
13814            (
13815                "exited with None and no usable output",
13816                "process exited with no usable output (exit code unavailable)",
13817            ),
13818        ] {
13819            let previous = FailClass::Other(failure_signature(old));
13820            let current = FailClass::Other(failure_signature(new));
13821            assert_eq!(previous, current, "resumption retains the failure class");
13822            assert!(!should_hand_over(Some(&previous), &current));
13823        }
13824        assert_eq!(
13825            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13826            failure_signature("cannot open /var/x.txt: denied")
13827        );
13828        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13829    }
13830
13831    #[test]
13832    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13833        let other = FailClass::Other("x".into());
13834        assert!(should_hand_over(None, &FailClass::Quota));
13835        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13836        assert!(should_hand_over(
13837            Some(&FailClass::Timeout),
13838            &FailClass::Timeout
13839        ));
13840        assert!(should_hand_over(None, &other));
13841        assert!(!should_hand_over(Some(&other), &other));
13842        assert!(should_hand_over(
13843            Some(&other),
13844            &FailClass::Other("y".into())
13845        ));
13846        // A quota or timeout in between ends the run of identical failures.
13847        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13848        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13849    }
13850
13851    #[test]
13852    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13853        let a = SeatState::new("judge-1", "alpha", 7);
13854        let b = handover_seat("judge-1", "beta", 7);
13855        assert_ne!(a.claude_session, b.claude_session);
13856        assert_eq!(b.turns, 0);
13857    }
13858
13859    #[test]
13860    fn a_timeout_is_classified_apart_from_other_failures() {
13861        assert_eq!(
13862            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13863            Some(FailClass::Timeout)
13864        );
13865        assert!(matches!(
13866            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13867            Some(FailClass::Other(_))
13868        ));
13869    }
13870
13871    #[test]
13872    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13873        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13874        let tried = BTreeSet::from(["beta".to_owned()]);
13875        // beta sits at index 1; the next candidate is gamma, never alpha —
13876        // which is very likely a different candidate slot's own agent.
13877        let next = next_untried_in_roster(&roster, 1, &tried);
13878        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13879    }
13880
13881    #[test]
13882    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13883        let roster = vec![spec("alpha"), spec("beta")];
13884        let tried = BTreeSet::from(["beta".to_owned()]);
13885        // beta is the roster's last entry: nothing follows it, and alpha —
13886        // earlier in the roster, almost certainly a different candidate
13887        // slot's own agent — must not be reached by wrapping back to it.
13888        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13889    }
13890
13891    #[test]
13892    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13893        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13894        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13895        // beta (index 1) and gamma (index 2, the only entry after it) have
13896        // both been tried; alpha (index 0) never has, but it comes before
13897        // beta's own position, so there is nothing further for this seat.
13898        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13899    }
13900
13901    #[test]
13902    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13903        let roster = vec![spec("a"), spec("a"), spec("b")];
13904        let tried = BTreeSet::from(["a".to_owned()]);
13905        let next = next_untried_in_roster(&roster, 0, &tried);
13906        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13907    }
13908
13909    #[test]
13910    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13911        let roster = vec![spec("a"), spec("b")];
13912        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13913        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13914    }
13915
13916    #[test]
13917    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13918        let dir = tempfile::tempdir().unwrap();
13919        let bay = dir.path().join("ffff");
13920
13921        // Not there yet: nothing to do, nothing to panic on.
13922        remove_if_empty(&bay);
13923        assert!(!bay.exists());
13924
13925        // Something still inside - the winner's worktree, or a stray file -
13926        // keeps the directory standing.
13927        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13928        remove_if_empty(&bay);
13929        assert!(bay.exists(), "non-empty directory must survive");
13930
13931        // Once the last entry is gone, so is the directory itself.
13932        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13933        remove_if_empty(&bay);
13934        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13935    }
13936
13937    // `round_is_clean` is the exact decision this task fixed: a round with a
13938    // seat that never answered must not read the same as a round every seat
13939    // actually reviewed. These are deterministic and process-free by design —
13940    // the equivalent end-to-end check (a real reviewer timing out under a
13941    // live graph run) is a genuine race against wall-clock contention, and a
13942    // spawn slow enough to blow even a generous budget under a loaded test
13943    // run must not turn this specific regression check flaky.
13944
13945    #[test]
13946    fn a_full_panel_that_found_nothing_is_clean() {
13947        assert!(round_is_clean(
13948            0,
13949            true,
13950            2,
13951            2,
13952            0,
13953            IncompleteReviewPolicy::Block
13954        ));
13955    }
13956
13957    #[test]
13958    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13959        assert!(!round_is_clean(
13960            0,
13961            true,
13962            1,
13963            2,
13964            0,
13965            IncompleteReviewPolicy::Block
13966        ));
13967    }
13968
13969    #[test]
13970    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13971        assert!(!round_is_clean(
13972            1,
13973            true,
13974            1,
13975            2,
13976            0,
13977            IncompleteReviewPolicy::Warn
13978        ));
13979    }
13980
13981    #[test]
13982    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13983        assert!(round_is_clean(
13984            0,
13985            true,
13986            1,
13987            2,
13988            0,
13989            IncompleteReviewPolicy::Warn
13990        ));
13991    }
13992
13993    #[test]
13994    fn a_full_panel_with_an_open_finding_is_not_clean() {
13995        assert!(!round_is_clean(
13996            1,
13997            true,
13998            2,
13999            2,
14000            0,
14001            IncompleteReviewPolicy::Block
14002        ));
14003    }
14004
14005    #[test]
14006    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14007        assert!(!round_is_clean(
14008            0,
14009            false,
14010            2,
14011            2,
14012            0,
14013            IncompleteReviewPolicy::Block
14014        ));
14015    }
14016
14017    // The stall this task closes: under the default `block` policy, a seat
14018    // missing only because it was rate limited must not force a wait for a
14019    // session limit that will not lift by the next round. `round_is_clean`
14020    // is where that quorum carve-out lives; the review loop around it never
14021    // changes what a reviewer's vote or a finding's severity means.
14022
14023    #[test]
14024    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14025        // 1 of 2 answered, and the one missing was quota'd — the exact
14026        // "review-2 rate limited (quota)" shape from the field report.
14027        assert!(round_is_clean(
14028            0,
14029            true,
14030            1,
14031            2,
14032            1,
14033            IncompleteReviewPolicy::Block
14034        ));
14035    }
14036
14037    #[test]
14038    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14039        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14040        // not a quota loss (`quota_missing` stays 0) — worth another try.
14041        assert!(!round_is_clean(
14042            0,
14043            true,
14044            1,
14045            2,
14046            0,
14047            IncompleteReviewPolicy::Block
14048        ));
14049    }
14050
14051    #[test]
14052    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14053        assert!(!round_is_clean(
14054            1,
14055            true,
14056            1,
14057            2,
14058            1,
14059            IncompleteReviewPolicy::Block
14060        ));
14061        assert!(!round_is_clean(
14062            0,
14063            false,
14064            1,
14065            2,
14066            1,
14067            IncompleteReviewPolicy::Block
14068        ));
14069    }
14070
14071    #[test]
14072    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14073        // Every seat quota'd, nobody answered: there is no panel to decide
14074        // on, so this must fall through to the existing block-and-retry
14075        // fallback rather than call an unreviewed patch clean.
14076        assert!(!round_is_clean(
14077            0,
14078            true,
14079            0,
14080            2,
14081            2,
14082            IncompleteReviewPolicy::Block
14083        ));
14084    }
14085
14086    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14087        CommandOutcome {
14088            command: "test".to_owned(),
14089            code,
14090            output_tail: String::new(),
14091            duration_ms: 0,
14092            resource_blocked,
14093        }
14094    }
14095
14096    #[cfg(unix)]
14097    #[tokio::test]
14098    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14099        ask_test_home();
14100        let temp = tempfile::tempdir().unwrap();
14101        let mut runner = runner_at(RunStatus::Gating);
14102        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14103        let commands = vec![
14104            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14105            "touch must-not-run".to_owned(),
14106        ];
14107        let (outcomes, retried, pids) = run_e2e_with_retry(
14108            &mut runner.state,
14109            &shell,
14110            &commands,
14111            temp.path(),
14112            Duration::from_secs(5),
14113            "fixture",
14114        )
14115        .await;
14116        assert_eq!(outcomes.len(), 1);
14117        assert!(outcomes[0].resource_blocked);
14118        assert!(!outcomes[0].build_failed());
14119        assert!(!retried);
14120        assert!(pids.is_empty());
14121        assert!(!temp.path().join("must-not-run").exists());
14122        assert_eq!(
14123            runner.state.resource_block.as_ref().unwrap().kind,
14124            "incremental cache admission"
14125        );
14126    }
14127
14128    #[test]
14129    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14130        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14131        assert!(
14132            !verify_inconclusive(&[outcome(Some(1), false)]),
14133            "an ordinary failure is still evidence about the patch"
14134        );
14135        assert!(verify_inconclusive(&[outcome(None, true)]));
14136        assert!(
14137            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14138            "one inconclusive outcome taints the whole batch"
14139        );
14140        assert!(!verify_inconclusive(&[]));
14141    }
14142
14143    #[tokio::test]
14144    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14145        // Alive for the first two checks, then dead - confirms the loop
14146        // actually re-polls rather than deciding once and sleeping out the
14147        // ceiling regardless.
14148        let calls = std::sync::atomic::AtomicUsize::new(0);
14149        let started = Instant::now();
14150        wait_for_pids_with(
14151            &[123],
14152            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14153            Duration::from_millis(5),
14154            Duration::from_secs(5),
14155        )
14156        .await;
14157        assert!(
14158            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14159            "must keep checking rather than deciding on the first answer"
14160        );
14161        assert!(
14162            started.elapsed() < Duration::from_secs(1),
14163            "must return the moment it is confirmed dead, not wait out the ceiling"
14164        );
14165    }
14166
14167    #[tokio::test]
14168    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14169        let started = Instant::now();
14170        wait_for_pids_with(
14171            &[123],
14172            |_| true, // never reports dead
14173            Duration::from_millis(5),
14174            Duration::from_millis(30),
14175        )
14176        .await;
14177        let elapsed = started.elapsed();
14178        assert!(
14179            elapsed >= Duration::from_millis(30),
14180            "must not give up before its own ceiling: {elapsed:?}"
14181        );
14182        assert!(
14183            elapsed < Duration::from_secs(1),
14184            "must not wait past its own ceiling either: {elapsed:?}"
14185        );
14186    }
14187
14188    #[tokio::test]
14189    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14190        let started = Instant::now();
14191        wait_for_pids_with(
14192            &[],
14193            |_| true,
14194            Duration::from_secs(5),
14195            Duration::from_secs(5),
14196        )
14197        .await;
14198        assert!(
14199            started.elapsed() < Duration::from_millis(200),
14200            "an empty pid list has nothing to confirm"
14201        );
14202    }
14203
14204    // `review_conclusion` is the exact decision the review hand-off task
14205    // fixed: a round budget spent (or a tree that stopped moving) must not
14206    // collapse into `Blocked` regardless of what verification actually
14207    // said. Deterministic and process-free for the same reason the
14208    // `round_is_clean` family above is.
14209    fn review_round(
14210        clean: bool,
14211        blocking: usize,
14212        answered: usize,
14213        expected: usize,
14214        progressed: bool,
14215        e2e_ok: bool,
14216    ) -> ReviewRound {
14217        ReviewRound {
14218            round: 1,
14219            head: "h".to_owned(),
14220            verified_head: None,
14221            verified_at: None,
14222            reviews: Vec::new(),
14223            e2e: vec![CommandOutcome {
14224                command: "test".to_owned(),
14225                code: Some(if e2e_ok { 0 } else { 1 }),
14226                output_tail: String::new(),
14227                duration_ms: 0,
14228                resource_blocked: false,
14229            }],
14230            verify_retried: false,
14231            e2e_deferred: false,
14232            e2e_defer_reason: None,
14233            fix: None,
14234            blocking,
14235            answered,
14236            expected,
14237            clean,
14238            progressed,
14239            vote_split: false,
14240            reconsideration: Vec::new(),
14241            verdict: None,
14242        }
14243    }
14244
14245    #[test]
14246    fn review_conclusion_is_none_when_nothing_has_run() {
14247        assert_eq!(review_conclusion(&[], 3), None);
14248    }
14249
14250    #[test]
14251    fn review_conclusion_is_none_while_rounds_remain() {
14252        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14253        assert_eq!(review_conclusion(&rounds, 3), None);
14254    }
14255
14256    #[test]
14257    fn review_conclusion_is_gating_once_a_round_is_clean() {
14258        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14259        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14260    }
14261
14262    #[test]
14263    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14264        let rounds = vec![
14265            review_round(false, 1, 2, 2, true, true),
14266            review_round(false, 1, 2, 2, true, true),
14267        ];
14268        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14269    }
14270
14271    #[test]
14272    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14273        let rounds = vec![
14274            review_round(false, 1, 2, 2, true, true),
14275            review_round(false, 1, 2, 2, true, false),
14276        ];
14277        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14278    }
14279
14280    #[test]
14281    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14282        // Magi never got a command to run against this round's own head — a
14283        // resource-blocked attempt, not a red one — so this must never
14284        // settle on `Blocked` the way a genuine e2e failure would. `None`
14285        // here is what tells `Runner::review_loop` to retry the check
14286        // itself rather than trust this cheap recomputation with a verdict
14287        // it cannot actually produce.
14288        let mut blocked = review_round(false, 1, 2, 2, true, false);
14289        blocked.e2e[0].resource_blocked = true;
14290        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14291        assert_eq!(review_conclusion(&rounds, 2), None);
14292    }
14293
14294    #[test]
14295    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14296        // Missing input, not a verified tree — never a hand-off candidate.
14297        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14298        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14299    }
14300
14301    #[test]
14302    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14303        let rounds = vec![
14304            review_round(false, 1, 2, 2, false, true),
14305            review_round(false, 1, 2, 2, false, true),
14306        ];
14307        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14308    }
14309
14310    fn secs(n: u64) -> Duration {
14311        Duration::from_secs(n)
14312    }
14313
14314    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14315    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14316    #[tokio::test]
14317    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14318        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14319        if std::env::var_os(CHILD).is_none() {
14320            // Other tests reconcile every run in the process-global home,
14321            // briefly taking owner locks. This ancestry test expects an idle
14322            // predecessor, so give it its own process and pinned test home.
14323            let output = std::process::Command::new(std::env::current_exe().unwrap())
14324                .args([
14325                    "--exact",
14326                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14327                    "--nocapture",
14328                ])
14329                .env(CHILD, "1")
14330                .env_remove("MAGI_WORKER_HOME")
14331                .env_remove("MAGI_WORKER_ID")
14332                .output()
14333                .unwrap();
14334            assert!(
14335                output.status.success()
14336                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14337                "{}\n{}",
14338                String::from_utf8_lossy(&output.stdout),
14339                String::from_utf8_lossy(&output.stderr)
14340            );
14341            return;
14342        }
14343        ask_test_home();
14344        let tmp = tempfile::tempdir().unwrap();
14345        let repo = tmp.path().join("repo");
14346        std::fs::create_dir_all(&repo).unwrap();
14347        init_repo(&repo);
14348        let g = |args: &[&str]| {
14349            let out = std::process::Command::new("git")
14350                .args(args)
14351                .current_dir(&repo)
14352                .quiet()
14353                .output()
14354                .unwrap();
14355            assert!(
14356                out.status.success(),
14357                "{}",
14358                String::from_utf8_lossy(&out.stderr)
14359            );
14360        };
14361        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14362        g(&["checkout", "-b", "historical"]);
14363        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14364        g(&["add", "."]);
14365        g(&["commit", "-m", "old alternative"]);
14366        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14367        g(&["checkout", "-b", "magi/task-selected", &base]);
14368        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14369        g(&["add", "."]);
14370        g(&["commit", "-m", "measurement infrastructure"]);
14371        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14372        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14373        g(&["add", "."]);
14374        g(&["commit", "-m", "dependent implementation"]);
14375        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14376        g(&["checkout", "main"]);
14377        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14378        g(&["add", "."]);
14379        g(&["commit", "-m", "unrelated base change"]);
14380        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14381        let origin = tmp.path().join("origin.git");
14382        g(&[
14383            "clone",
14384            "--bare",
14385            repo.to_str().unwrap(),
14386            origin.to_str().unwrap(),
14387        ]);
14388        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14389        g(&["fetch", "origin"]);
14390        let selection = crate::pr_recovery::Selection {
14391            url: "https://example.test/project/pull/1".into(),
14392            branch: "magi/task-selected".into(),
14393            head: head.clone(),
14394            source_head: None,
14395            source_run: None,
14396            source_proof: None,
14397            disposition: Some(crate::pr_recovery::Disposition::Review),
14398            instruction: "preserve measured behavior".into(),
14399            approved_at: jiff::Timestamp::now(),
14400        };
14401        let mut template = runner_at(RunStatus::Reviewing);
14402        template.state.repo = repo.clone();
14403        template.state.base_commit = current.clone();
14404        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14405        template.state.origin = Some(crate::run::Origin::queue("selected"));
14406        template.state.pr_head = Some(selection.branch.clone());
14407        template.state.pr_recovery = Some(selection.clone());
14408        let mut runner =
14409            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14410                .await
14411                .unwrap();
14412        let wt = runner.state.winner().unwrap().worktree.clone();
14413        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14414        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14415        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14416        assert!(runner.state.seeds.is_empty());
14417        assert_eq!(runner.state.instruction, "preserve measured behavior");
14418        runner.sync_to_base().await.unwrap();
14419        assert_eq!(
14420            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14421            "prerequisite"
14422        );
14423        assert_eq!(
14424            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14425            "measured implementation"
14426        );
14427        assert_eq!(
14428            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14429            "current main change"
14430        );
14431        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14432        assert_eq!(
14433            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14434            head
14435        );
14436
14437        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14438        let mut partial = runner_at(RunStatus::Reviewing);
14439        partial.state.repo = repo.clone();
14440        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14441        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14442            run: Some("exit 1".into()),
14443            ..Default::default()
14444        }];
14445        let selection = runner.state.pr_recovery.clone().unwrap();
14446        partial.state.pr_recovery = Some(selection.clone());
14447        partial.state.pr_head = Some(selection.branch.clone());
14448        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14449        let id = partial.state.id.clone();
14450        let error = match Runner::open_selected(
14451            &repo,
14452            partial.state,
14453            partial.roles,
14454            1,
14455            runner.state.base_commit.clone(),
14456            selection,
14457        )
14458        .await
14459        {
14460            Ok(_) => panic!("setup must fail"),
14461            Err(e) => e,
14462        };
14463        assert_eq!(
14464            error
14465                .downcast_ref::<crate::run::StartupFailure>()
14466                .unwrap()
14467                .run,
14468            id
14469        );
14470        let failed = RunState::load(&id).unwrap();
14471        assert_eq!(failed.status, RunStatus::Failed);
14472        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14473        assert!(Runner::resume(&id).is_err());
14474        runner.state.status = RunStatus::Ready;
14475        runner.state.driver_exited = true;
14476        runner.state.save().unwrap();
14477        let mut retry = runner_at(RunStatus::Reviewing);
14478        retry.state.repo = repo.clone();
14479        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14480        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14481        reselected.approved_at = jiff::Timestamp::now();
14482        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14483        retry.state.pr_head = Some(reselected.branch.clone());
14484        retry.state.pr_recovery = Some(reselected.clone());
14485        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14486            .await
14487            .unwrap();
14488        let restarted = Runner::open_selected(
14489            &repo,
14490            retry.state,
14491            retry.roles,
14492            1,
14493            runner.state.base_commit.clone(),
14494            reselected,
14495        )
14496        .await
14497        .unwrap();
14498        assert_eq!(
14499            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14500                .await
14501                .unwrap(),
14502            head
14503        );
14504        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14505        assert_eq!(
14506            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14507            head
14508        );
14509    }
14510
14511    #[tokio::test]
14512    #[cfg(unix)]
14513    async fn fallback_and_interrupted_restoration_require_coordination() {
14514        use std::os::fd::AsRawFd as _;
14515        ask_test_home();
14516        let tmp = tempfile::tempdir().unwrap();
14517        let mut runner = registered_runner(tmp.path()).await;
14518        runner.state.config.graph.timeout_resource_wait = 1;
14519        let winner = runner.state.candidates[0].clone();
14520        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14521        let leftover = winner.worktree.join("leftover");
14522        std::fs::write(&leftover, "untrusted edits").unwrap();
14523        let next = tmp.path().join("next-fixer");
14524        let mut failed = spec("first");
14525        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14526        let mut successor = spec("second");
14527        successor.command = vec![
14528            "sh".into(),
14529            "-c".into(),
14530            format!("touch '{}'; echo done", next.display()),
14531        ];
14532        let attempts = vec![
14533            (failed.clone(), "first".into()),
14534            (successor.clone(), "second".into()),
14535        ];
14536        let lock_path = runner.state.repo.join(".git/updater.lock");
14537        let external = std::fs::File::create(&lock_path).unwrap();
14538        assert_eq!(
14539            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14540            0
14541        );
14542        let artifacts = tmp.path().join("artifacts");
14543        let build = |spec, seat| SeatJob {
14544            spec,
14545            seat,
14546            cwd: winner.worktree.clone(),
14547            prompt: "fixture".into(),
14548            timeout: Duration::from_secs(10),
14549            allow_write: true,
14550            sessions: false,
14551            artifacts: artifacts.clone(),
14552            stem: "fix".into(),
14553            handover: None,
14554        };
14555        let (_, _, out) = runner
14556            .ask_fixer_among(
14557                attempts.clone(),
14558                "graft",
14559                None,
14560                Some((&winner.worktree, &head)),
14561                build,
14562            )
14563            .await;
14564        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14565        assert!(leftover.exists(), "contention must prevent clean");
14566        assert!(!next.exists());
14567        assert!(runner.state.handovers.is_empty());
14568        // Early interrupted-repair rollback uses the same protected route.
14569        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14570            from_run: "earlier".into(),
14571            generation: 1,
14572            mode: crate::recovery::Mode::VerifyRepair,
14573            reasons: Vec::new(),
14574            reviewed_head: None,
14575            rounds: Vec::new(),
14576            open_findings: Vec::new(),
14577            evidence: None,
14578            criteria_inherited: true,
14579            repair: crate::recovery::Repair::Started {
14580                before: head.clone(),
14581            },
14582        });
14583        assert!(
14584            runner
14585                .recovery_repair(&winner)
14586                .await
14587                .unwrap_err()
14588                .is::<crate::repo_lock::ResourceBlocked>()
14589        );
14590        assert!(matches!(
14591            runner.state.recovery.as_ref().unwrap().repair,
14592            crate::recovery::Repair::Started { .. }
14593        ));
14594        assert!(leftover.exists());
14595        drop(external);
14596        runner.recovery_repair(&winner).await.unwrap();
14597        assert!(!leftover.exists());
14598        assert!(matches!(
14599            runner.state.recovery.as_ref().unwrap().repair,
14600            crate::recovery::Repair::Done { .. }
14601        ));
14602        // A hard lock-open failure also leaves HEAD and untracked work intact.
14603        std::fs::remove_file(&lock_path).unwrap();
14604        std::fs::create_dir(&lock_path).unwrap();
14605        std::fs::write(&leftover, "untrusted edits").unwrap();
14606        let (_, _, out) = runner
14607            .ask_fixer_among(
14608                attempts,
14609                "graft",
14610                None,
14611                Some((&winner.worktree, &head)),
14612                build,
14613            )
14614            .await;
14615        assert!(matches!(out, AgentOutcome::Failed(_)));
14616        assert!(leftover.exists());
14617        assert!(!next.exists());
14618        assert_eq!(
14619            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14620            head
14621        );
14622    }
14623
14624    #[cfg(unix)]
14625    async fn registered_runner(root: &Path) -> Runner {
14626        repository_runner(root, crate::config::test_coordination()).await
14627    }
14628
14629    async fn repository_runner(
14630        root: &Path,
14631        coordination: Option<crate::config::Coordination>,
14632    ) -> Runner {
14633        ask_test_home();
14634        let repo = root.join("repo");
14635        std::fs::create_dir_all(&repo).unwrap();
14636        init_repo(&repo);
14637        if coordination.is_some() {
14638            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14639        }
14640        let mut runner = runner_at(RunStatus::Prep);
14641        runner.state.repo = repo.clone();
14642        runner.state.config.coordination = coordination;
14643        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14644        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14645        runner.state.config.graph.advise = false;
14646        runner.roles.implementers = vec![spec("implement")];
14647        runner.roles.judges = vec![spec("judge")];
14648        runner.prep().await.unwrap();
14649        runner
14650    }
14651
14652    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14653
14654    #[tokio::test]
14655    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14656        let tmp = tempfile::tempdir().unwrap();
14657        let mut runner = repository_runner(tmp.path(), None).await;
14658        let prompts = runner.state.config.prompts.clone();
14659        let run = runner.state.id.clone();
14660        let ctx = WaveCtx {
14661            carry_seats: false,
14662            pauses: None,
14663            run: &run,
14664            node: "advise",
14665            prompts: &prompts,
14666            cache: None,
14667            round: None,
14668        };
14669        let jobs: Vec<_> = ["alpha", "beta"]
14670            .into_iter()
14671            .map(|id| SeatJob {
14672                spec: scripted(id, AUTH_OUTAGE),
14673                seat: SeatState::new(id, id, 7),
14674                prompt: "propose".into(),
14675                cwd: runner.state.candidates[0].worktree.clone(),
14676                timeout: Duration::from_secs(10),
14677                allow_write: false,
14678                sessions: false,
14679                artifacts: agent::artifacts_dir(&runner.state.dir()),
14680                stem: id.into(),
14681                handover: None,
14682            })
14683            .collect();
14684        let mut losses = Vec::new();
14685        let results = ask_wave_with(
14686            jobs.clone(),
14687            runner.sem.clone(),
14688            2,
14689            &[],
14690            &ctx,
14691            &mut losses,
14692            &mut runner.state,
14693            &|_: &str| Ok(()),
14694        )
14695        .await;
14696        assert!(
14697            results
14698                .iter()
14699                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14700        );
14701        assert_eq!(runner.state.quota.len(), 2);
14702        assert!(
14703            runner
14704                .state
14705                .quota
14706                .iter()
14707                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14708        );
14709        assert!(!runner.state.advise_attempted);
14710        runner.stop_for_resource().unwrap();
14711        assert_eq!(runner.state.status, RunStatus::Blocked);
14712        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14713        assert_eq!(
14714            runner.state.resource_block.as_ref().unwrap().kind,
14715            "agent authentication"
14716        );
14717        // A machine loss followed by an ordinary failure is not evidence
14718        // that the whole node was unavailable, and must not earn a refund.
14719        runner.state.resource_block = None;
14720        runner.state.quota.clear();
14721        let mut mixed = jobs;
14722        mixed[1].spec.command = vec!["false".into()];
14723        let results = ask_wave_with(
14724            mixed,
14725            runner.sem.clone(),
14726            0,
14727            &[],
14728            &ctx,
14729            &mut losses,
14730            &mut runner.state,
14731            &|_: &str| Ok(()),
14732        )
14733        .await;
14734        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14735        assert!(runner.state.resource_block.is_none());
14736        assert_eq!(losses.len(), 1);
14737    }
14738
14739    #[tokio::test]
14740    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14741        let tmp = tempfile::tempdir().unwrap();
14742        let mut runner = repository_runner(tmp.path(), None).await;
14743        let implementer = scripted("implement", AUTH_OUTAGE);
14744        runner.roles.implementers = vec![implementer.clone()];
14745        runner.state.config.agents = vec![implementer];
14746        runner.implement().await.unwrap();
14747        assert_eq!(runner.state.status, RunStatus::Blocked);
14748        assert_eq!(runner.state.quota.len(), 1);
14749        assert_eq!(
14750            runner.state.quota[0].cause,
14751            agent::QuotaCause::Authentication
14752        );
14753        let candidate = &runner.state.candidates[0];
14754        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14755        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14756    }
14757
14758    #[tokio::test]
14759    #[cfg(unix)]
14760    async fn parked_implementer_persists_completed_work() {
14761        for registered in [false, true] {
14762            let tmp = tempfile::tempdir().unwrap();
14763            let mut runner = registered_runner(tmp.path()).await;
14764            if !registered {
14765                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14766            }
14767            let marker = tmp.path().join("started");
14768            let release = tmp.path().join("release");
14769            let mut implementer = spec("implement");
14770            implementer.command = vec![
14771                "sh".into(),
14772                "-c".into(),
14773                format!(
14774                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14775                    marker.display(),
14776                    release.display()
14777                ),
14778            ];
14779            runner.roles.implementers = vec![implementer.clone()];
14780            runner.state.config.agents = vec![implementer];
14781            let pause = runner.pause.clone();
14782            let (result, ()) = tokio::join!(runner.implement(), async {
14783                tokio::time::timeout(Duration::from_secs(10), async {
14784                    while !marker.exists() {
14785                        tokio::time::sleep(Duration::from_millis(10)).await;
14786                    }
14787                })
14788                .await
14789                .unwrap();
14790                pause.park();
14791                std::fs::write(&release, "go").unwrap();
14792            });
14793            result.unwrap();
14794            let candidate = &runner.state.candidates[0];
14795            assert!(candidate.commits > 0, "{candidate:?}");
14796            assert!(candidate.summary.contains("saved-result"));
14797            assert_eq!(candidate.files, 1);
14798            let before = candidate.commits;
14799            runner.implement().await.unwrap();
14800            assert_eq!(runner.state.candidates[0].commits, before);
14801        }
14802    }
14803
14804    #[tokio::test]
14805    #[cfg(unix)]
14806    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14807        let tmp = tempfile::tempdir().unwrap();
14808        let mut runner = registered_runner(tmp.path()).await;
14809        let winner = runner.state.candidates[0].clone();
14810        let cache = tmp.path().join("cache");
14811        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14812        runner.state.config.graph.timeout_resource_wait = 0;
14813        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14814        let crate::cache::AcquireOutcome::Acquired(held) =
14815            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14816        else {
14817            panic!("cache must be free")
14818        };
14819        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14820            from_run: "earlier".into(),
14821            generation: 1,
14822            mode: crate::recovery::Mode::VerifyRepair,
14823            reasons: Vec::new(),
14824            reviewed_head: None,
14825            rounds: Vec::new(),
14826            open_findings: Vec::new(),
14827            evidence: None,
14828            criteria_inherited: true,
14829            repair: crate::recovery::Repair::NotStarted,
14830        });
14831        runner.recovery_repair(&winner).await.unwrap();
14832        assert!(matches!(
14833            runner.state.recovery.as_ref().unwrap().repair,
14834            crate::recovery::Repair::WaitingForRestore { .. }
14835        ));
14836        drop(held);
14837        runner.state.resource_block = None;
14838        runner.state.config.graph.timeout_resource_wait = 1;
14839        runner.recovery_repair(&winner).await.unwrap();
14840        assert!(matches!(
14841            runner.state.recovery.as_ref().unwrap().repair,
14842            crate::recovery::Repair::Done { .. }
14843        ));
14844        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14845    }
14846
14847    #[tokio::test]
14848    #[cfg(unix)]
14849    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14850        use std::os::fd::AsRawFd as _;
14851        for graft in [false, true] {
14852            let tmp = tempfile::tempdir().unwrap();
14853            let mut runner = registered_runner(tmp.path()).await;
14854            runner.state.config.graph.timeout_resource_wait = 1;
14855            let winner = runner.state.candidates[0].clone();
14856            let marker = tmp.path().join("fixer-started");
14857            let mut fixer = spec("fixer");
14858            fixer.command = vec![
14859                "sh".into(),
14860                "-c".into(),
14861                format!("echo started >> '{}'; echo done", marker.display()),
14862            ];
14863            runner.roles.fixer = Some(vec![fixer]);
14864            if graft {
14865                let mut loser = winner.clone();
14866                loser.label = 'B';
14867                loser.index = 1;
14868                runner.state.candidates.push(loser);
14869                runner.state.config.graph.graft_runner_up = true;
14870                runner.state.tally = Some(Tally {
14871                    first_choice: BTreeMap::new(),
14872                    borda: BTreeMap::new(),
14873                    winner: winner.label,
14874                    rankings: 1,
14875                    unanimous_initial: true,
14876                    deliberated: false,
14877                    changed_votes: 0,
14878                    unanimous_final: true,
14879                    tie_break: None,
14880                    judges: 1,
14881                    present: 1,
14882                    quorum: 1,
14883                    met_quorum: true,
14884                    uncontested: None,
14885                });
14886                runner.state.judgements.push(crate::run::Judgement {
14887                    judge: 1,
14888                    seat: "judge-1".into(),
14889                    agent: "judge".into(),
14890                    ranking: vec![winner.label, 'B'],
14891                    reasons: BTreeMap::new(),
14892                    confidence: None,
14893                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14894                    order: vec![0, 1],
14895                    failed: None,
14896                    duration_ms: 0,
14897                });
14898            } else {
14899                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14900                    from_run: "earlier".into(),
14901                    generation: 1,
14902                    mode: crate::recovery::Mode::VerifyRepair,
14903                    reasons: Vec::new(),
14904                    reviewed_head: None,
14905                    rounds: Vec::new(),
14906                    open_findings: Vec::new(),
14907                    evidence: None,
14908                    criteria_inherited: true,
14909                    repair: crate::recovery::Repair::NotStarted,
14910                });
14911            }
14912            let reader = crate::repo_lock::SeatLease::for_checkout(
14913                &winner.worktree,
14914                false,
14915                crate::config::test_coordination().as_ref(),
14916            )
14917            .await
14918            .unwrap();
14919            let pause = runner.pause.clone();
14920            let (result, ()) = tokio::join!(
14921                async {
14922                    if graft {
14923                        runner.graft().await
14924                    } else {
14925                        runner.recovery_repair(&winner).await
14926                    }
14927                },
14928                async {
14929                    tokio::time::sleep(Duration::from_millis(100)).await;
14930                    pause.park();
14931                }
14932            );
14933            result.unwrap();
14934            assert!(!marker.exists(), "a blocked seat must not spawn");
14935            // Exercise the persisted state, as a binary replacement/resume does.
14936            runner.state = RunState::load(&runner.state.id).unwrap();
14937            if graft {
14938                assert_eq!(
14939                    runner.state.graft.as_ref().unwrap().outcome,
14940                    GraftOutcome::WaitingForRestore
14941                );
14942            } else {
14943                assert!(matches!(
14944                    runner.state.recovery.as_ref().unwrap().repair,
14945                    crate::recovery::Repair::WaitingForRestore { .. }
14946                ));
14947            }
14948            drop(reader);
14949            runner.pause = Pause::new();
14950            runner.state.resource_block = None;
14951            // Deferred restoration must still respect an external lock owner.
14952            let external =
14953                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14954            assert_eq!(
14955                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14956                0
14957            );
14958            let leftover = winner.worktree.join("leftover");
14959            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14960            let result = if graft {
14961                runner.graft().await
14962            } else {
14963                runner.recovery_repair(&winner).await
14964            };
14965            assert!(
14966                result
14967                    .unwrap_err()
14968                    .is::<crate::repo_lock::ResourceBlocked>()
14969            );
14970            assert!(leftover.exists());
14971            assert!(!marker.exists());
14972            drop(external);
14973            runner.state.resource_block = None;
14974            let result = if graft {
14975                runner.graft().await
14976            } else {
14977                runner.recovery_repair(&winner).await
14978            };
14979            result.unwrap();
14980            assert!(!leftover.exists(), "restore must precede retry");
14981            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14982            if graft {
14983                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14984            } else {
14985                assert!(matches!(
14986                    runner.state.recovery.as_ref().unwrap().repair,
14987                    crate::recovery::Repair::Done { .. }
14988                ));
14989            }
14990        }
14991    }
14992
14993    #[tokio::test]
14994    #[cfg(unix)]
14995    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
14996        use std::os::fd::AsRawFd as _;
14997        let tmp = tempfile::tempdir().unwrap();
14998        let mut runner = registered_runner(tmp.path()).await;
14999        runner.state.config.graph.timeout_resource_wait = 1;
15000        let winner = runner.state.candidates[0].clone();
15001        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15002        runner.state.tally = Some(Tally {
15003            first_choice: BTreeMap::new(),
15004            borda: BTreeMap::new(),
15005            winner: winner.label,
15006            rankings: 0,
15007            unanimous_initial: false,
15008            deliberated: false,
15009            changed_votes: 0,
15010            unanimous_final: false,
15011            tie_break: None,
15012            judges: 0,
15013            present: 0,
15014            quorum: 0,
15015            met_quorum: true,
15016            uncontested: Some("fixture".into()),
15017        });
15018        runner.state.graft = Some(GraftRecord {
15019            pre: pre.clone(),
15020            post: None,
15021            lines: None,
15022            limit: None,
15023            agent: None,
15024            outcome: GraftOutcome::Running,
15025            requested: Vec::new(),
15026            adoption: None,
15027            adoption_error: None,
15028        });
15029        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15030        git::commit_all(&winner.worktree, "unfinished graft")
15031            .await
15032            .unwrap();
15033        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15034        let leftover = winner.worktree.join("leftover");
15035        std::fs::write(&leftover, "untrusted edits").unwrap();
15036        let path = runner.state.repo.join(".git/updater.lock");
15037        let external = std::fs::File::create(&path).unwrap();
15038        assert_eq!(
15039            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15040            0
15041        );
15042        assert!(
15043            runner
15044                .graft()
15045                .await
15046                .unwrap_err()
15047                .is::<crate::repo_lock::ResourceBlocked>()
15048        );
15049        assert!(matches!(
15050            runner.state.graft.as_ref().unwrap().outcome,
15051            GraftOutcome::Running
15052        ));
15053        assert_eq!(
15054            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15055            unfinished
15056        );
15057        assert!(leftover.exists());
15058        drop(external);
15059        std::fs::remove_file(&path).unwrap();
15060        std::fs::create_dir(&path).unwrap();
15061        assert!(runner.graft().await.is_err());
15062        assert!(matches!(
15063            runner.state.graft.as_ref().unwrap().outcome,
15064            GraftOutcome::Running
15065        ));
15066        assert_eq!(
15067            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15068            unfinished
15069        );
15070        assert!(leftover.exists());
15071        std::fs::remove_dir(&path).unwrap();
15072        runner.graft().await.unwrap();
15073        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15074        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15075        assert!(!leftover.exists());
15076    }
15077
15078    #[tokio::test]
15079    #[cfg(unix)]
15080    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15081        use std::os::fd::AsRawFd as _;
15082        let tmp = tempfile::tempdir().unwrap();
15083        let mut runner = registered_runner(tmp.path()).await;
15084        runner.state.prep_complete = Some(false);
15085        runner.state.config.graph.timeout_resource_wait = 1;
15086        runner.state.save().unwrap();
15087        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15088        assert_eq!(
15089            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15090            0
15091        );
15092        runner.execute().await.unwrap();
15093        let disk = RunState::load(&runner.state.id).unwrap();
15094        assert!(disk.driver_exited);
15095        assert_eq!(disk.driver_pid, None);
15096        assert_eq!(runner.state.status, RunStatus::Blocked);
15097        assert_eq!(
15098            runner.state.resource_block.as_ref().unwrap().kind,
15099            "repository coordination"
15100        );
15101        assert!(runner.state.prep_failure.is_none());
15102        assert!(runner.state.seats.is_empty());
15103        runner.pause.park();
15104        // Include headroom for Git subprocesses under parallel test load;
15105        // the repository contention deadline itself remains one second.
15106        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15107            .await
15108            .unwrap()
15109            .unwrap();
15110        assert!(runner.state.parked);
15111        assert!(runner.state.prep_failure.is_none());
15112        let disk = RunState::load(&runner.state.id).unwrap();
15113        assert!(disk.driver_exited);
15114        assert_eq!(disk.driver_pid, None);
15115    }
15116
15117    #[tokio::test]
15118    #[cfg(unix)]
15119    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15120        use std::os::fd::AsRawFd as _;
15121        let tmp = tempfile::tempdir().unwrap();
15122        let mut runner = registered_runner(tmp.path()).await;
15123        runner.state.prep_complete = Some(false);
15124        runner.state.config.graph.timeout_resource_wait = 30;
15125        runner.state.save().unwrap();
15126        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15127        assert_eq!(
15128            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15129            0
15130        );
15131        assert!(
15132            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15133                .await
15134                .is_err()
15135        );
15136        let disk = RunState::load(&runner.state.id).unwrap();
15137        assert!(disk.driver_exited);
15138        assert_eq!(disk.driver_pid, None);
15139        assert!(disk.status.resumable());
15140        assert!(runner.state.driver_exited);
15141        assert_eq!(runner.state.driver_pid, None);
15142        runner.state.save().unwrap();
15143        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15144    }
15145
15146    #[tokio::test]
15147    #[cfg(unix)]
15148    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15149        let tmp = tempfile::tempdir().unwrap();
15150        let mut runner = registered_runner(tmp.path()).await;
15151        let winner = runner.state.candidates[0].clone();
15152        let cache = tmp.path().join("cache");
15153        let started = tmp.path().join("pre-gate-started");
15154        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15155        runner.state.config.verify.gate =
15156            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15157        let home = crate::run::home();
15158        let owner =
15159            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15160        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15161            crate::cache::AcquireOutcome::Acquired(g) => g,
15162            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15163        };
15164        let mut gate = Box::pin(runner.run_gate(&winner));
15165        assert!(
15166            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15167                .await
15168                .is_err()
15169        );
15170        assert!(!started.exists(), "pre-gate must wait for the cache too");
15171        // Simulate the wave that already owns the cache entering invoke.
15172        // It must obtain the editor reservation and finish without waiting
15173        // for the gate that is queued behind its cache.
15174        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15175        let editor = crate::repo_lock::SeatLease::checkout_with(
15176            &winner.worktree,
15177            true,
15178            crate::config::test_coordination().as_ref(),
15179            &wait,
15180        )
15181        .await
15182        .unwrap();
15183        drop(editor);
15184        guard.release();
15185        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15186            .await
15187            .unwrap()
15188            .unwrap();
15189        drop(gate);
15190        assert!(outcomes.iter().all(CommandOutcome::ok));
15191        assert!(started.exists());
15192    }
15193
15194    #[tokio::test]
15195    #[cfg(unix)]
15196    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15197        use std::os::fd::AsRawFd as _;
15198
15199        ask_test_home();
15200        let tmp = tempfile::tempdir().unwrap();
15201        let repo = tmp.path().join("repo");
15202        std::fs::create_dir_all(&repo).unwrap();
15203        init_repo(&repo);
15204        let common = repo.join(".git");
15205        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15206        let mut runner = runner_at(RunStatus::Prep);
15207        runner.state.repo = repo.clone();
15208        runner.state.config.coordination = crate::config::test_coordination();
15209        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15210        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15211        runner.state.config.graph.advise = false;
15212        runner.roles.implementers = vec![spec("implement")];
15213        runner.roles.judges = vec![spec("judge")];
15214        runner.prep().await.unwrap();
15215        let winner = runner.state.candidates[0].clone();
15216        // Emulate edits already left by a completed seat. The fixture CLI
15217        // only reports completion; it performs no repository operations.
15218        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15219        let completed = tmp.path().join("seat-completed");
15220        runner.roles.implementers[0].command = vec![
15221            "sh".into(),
15222            "-c".into(),
15223            format!("touch '{}'; echo completed", completed.display()),
15224        ];
15225        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15226        assert_eq!(
15227            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15228            0
15229        );
15230        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15231        let mut implement = Box::pin(runner.implement());
15232        tokio::select! {
15233            result = &mut implement => panic!("rescue must wait: {result:?}"),
15234            result = tokio::time::timeout(Duration::from_secs(10), async {
15235                while !completed.exists() {
15236                    tokio::time::sleep(Duration::from_millis(20)).await;
15237                }
15238            }) => result.unwrap(),
15239        }
15240        assert!(
15241            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15242                .await
15243                .is_err()
15244        );
15245        assert_eq!(
15246            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15247            before
15248        );
15249        assert_eq!(
15250            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15251            0
15252        );
15253        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15254            .await
15255            .unwrap()
15256            .unwrap();
15257        drop(implement);
15258        assert_ne!(
15259            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15260            before
15261        );
15262        assert!(git::is_clean(&winner.worktree).await.unwrap());
15263
15264        let started = tmp.path().join("pre-gate-started");
15265        let proceed = tmp.path().join("pre-gate-proceed");
15266        let checked = tmp.path().join("gate-ran");
15267        runner.state.config.verify.pre_gate = vec![format!(
15268            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15269            started.display(),
15270            proceed.display()
15271        )];
15272        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15273        assert_eq!(
15274            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15275            0
15276        );
15277        let mut gate = Box::pin(runner.run_gate(&winner));
15278        assert!(
15279            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15280                .await
15281                .is_err()
15282        );
15283        assert!(!started.exists());
15284        assert!(!checked.exists());
15285        assert!(git::is_clean(&winner.worktree).await.unwrap());
15286        assert_eq!(
15287            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15288            0
15289        );
15290        tokio::select! {
15291            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15292            result = tokio::time::timeout(Duration::from_secs(10), async {
15293                while !started.exists() {
15294                    tokio::time::sleep(Duration::from_millis(20)).await;
15295                }
15296            }) => result.unwrap(),
15297        }
15298        assert_eq!(
15299            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15300            -1,
15301            "mechanical edits must hold the real lock"
15302        );
15303        std::fs::write(proceed, "ready").unwrap();
15304        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15305            .await
15306            .unwrap()
15307            .unwrap();
15308        drop(gate);
15309        assert!(outcomes.iter().all(CommandOutcome::ok));
15310        assert!(checked.exists());
15311        assert!(runner.state.pre_gate_commit.is_some());
15312        assert!(git::is_clean(&winner.worktree).await.unwrap());
15313        assert_eq!(
15314            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15315            0,
15316            "gate completion must release coordination"
15317        );
15318        drop(outsider);
15319        std::fs::remove_file(common.join("updater.lock")).unwrap();
15320        std::fs::create_dir(common.join("updater.lock")).unwrap();
15321        std::fs::remove_file(&started).unwrap();
15322        std::fs::remove_file(&checked).unwrap();
15323        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15324        let error = runner.run_gate(&winner).await.unwrap_err();
15325        assert!(
15326            error
15327                .to_string()
15328                .contains("acquire mandatory repository lock")
15329        );
15330        assert!(
15331            !started.exists(),
15332            "failed acquisition must prevent pre-gate"
15333        );
15334        assert!(!checked.exists(), "failed acquisition must prevent gate");
15335        assert!(git::is_clean(&winner.worktree).await.unwrap());
15336        assert_eq!(
15337            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15338            head
15339        );
15340    }
15341
15342    #[tokio::test]
15343    #[cfg(unix)]
15344    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15345        use std::os::fd::AsRawFd as _;
15346        use std::time::Duration;
15347
15348        ask_test_home();
15349        let tmp = tempfile::tempdir().unwrap();
15350        let repo = tmp.path().join("repo");
15351        std::fs::create_dir_all(&repo).unwrap();
15352        init_repo(&repo);
15353        let common = repo.join(".git");
15354        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15355        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15356        assert_eq!(
15357            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15358            0
15359        );
15360        let mut runner = runner_at(RunStatus::Prep);
15361        runner.state.repo = repo.clone();
15362        runner.state.config.coordination = crate::config::test_coordination();
15363        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15364        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15365        runner.state.config.graph.advise = true;
15366        runner.state.config.graph.advisors = 1;
15367        runner.roles.implementers = vec![spec("implement")];
15368        runner.roles.judges = vec![spec("judge")];
15369        let root = runner.state.worktree_root();
15370        let started = tmp.path().join("setup-started");
15371        let proceed = tmp.path().join("setup-proceed");
15372        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15373            run: Some(format!(
15374                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15375                started.display(),
15376                proceed.display()
15377            )),
15378            ..Default::default()
15379        }];
15380        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15381        let config = git::git(&repo, &["config", "--local", "--list"])
15382            .await
15383            .unwrap();
15384        let mut prep = Box::pin(runner.prep());
15385        assert!(
15386            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15387                .await
15388                .is_err()
15389        );
15390        assert!(
15391            !root.exists(),
15392            "contention must prevent even checkout creation"
15393        );
15394        assert!(!started.exists(), "contention must prevent setup");
15395        assert_eq!(
15396            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15397            branches
15398        );
15399        assert_eq!(
15400            git::git(&repo, &["config", "--local", "--list"])
15401                .await
15402                .unwrap(),
15403            config
15404        );
15405        drop(lock);
15406        tokio::select! {
15407            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15408            result = tokio::time::timeout(Duration::from_secs(10), async {
15409                while !started.exists() {
15410                    tokio::time::sleep(Duration::from_millis(20)).await;
15411                }
15412            }) => result.unwrap(),
15413        }
15414        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15415        assert_eq!(
15416            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15417            -1
15418        );
15419        assert!(
15420            tokio::time::timeout(
15421                Duration::from_millis(100),
15422                crate::repo_lock::SeatLease::for_checkout(
15423                    &repo,
15424                    true,
15425                    crate::config::test_coordination().as_ref()
15426                )
15427            )
15428            .await
15429            .is_err(),
15430            "preparation must also exclude local editors"
15431        );
15432        std::fs::write(proceed, "ready").unwrap();
15433        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15434            .await
15435            .unwrap()
15436            .unwrap();
15437        drop(prep);
15438        assert_eq!(runner.state.prep_complete, Some(true));
15439        assert!(root.join("judge-1").exists());
15440        assert!(root.join("advisor-1").exists());
15441        assert_eq!(
15442            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15443            0
15444        );
15445    }
15446
15447    #[tokio::test]
15448    #[cfg(unix)]
15449    async fn preparation_lock_failure_prevents_repository_changes() {
15450        ask_test_home();
15451        let tmp = tempfile::tempdir().unwrap();
15452        let repo = tmp.path().join("repo");
15453        std::fs::create_dir_all(&repo).unwrap();
15454        init_repo(&repo);
15455        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15456        // A directory at the required lock path makes opening the real lock
15457        // fail, rather than merely exercising an invalid registration.
15458        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15459        let mut runner = runner_at(RunStatus::Prep);
15460        runner.state.repo = repo.clone();
15461        runner.state.config.coordination = crate::config::test_coordination();
15462        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15463        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15464        runner.roles.implementers = vec![spec("implement")];
15465        let error = runner.prep().await.unwrap_err();
15466        assert!(
15467            error
15468                .to_string()
15469                .contains("acquire mandatory repository lock")
15470        );
15471        assert!(!runner.state.worktree_root().exists());
15472        assert!(runner.state.candidates.is_empty());
15473        assert!(!repo.join(".git/config.worktree").exists());
15474        assert_eq!(
15475            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15476            "* main"
15477        );
15478    }
15479
15480    #[tokio::test]
15481    async fn preparation_records_creation_intent_before_git_can_fail() {
15482        ask_test_home();
15483        let tmp = tempfile::tempdir().unwrap();
15484        let repo = tmp.path().join("repo");
15485        std::fs::create_dir_all(&repo).unwrap();
15486        init_repo(&repo);
15487        let mut runner = runner_at(RunStatus::Prep);
15488        runner.state.repo = repo.clone();
15489        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15490        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15491        runner.state.config.blind.commit_msg_hook = false;
15492        runner.roles.implementers = vec![spec("one")];
15493        let label = blind::assign_labels(1, runner.state.seed)[0];
15494        let branch = runner.state.branch_for(label);
15495        // A conflicting branch makes Git refuse creation before a candidate
15496        // can be registered. The durable intent must still identify it.
15497        git::git(&repo, &["branch", &branch]).await.unwrap();
15498        assert!(runner.prep().await.is_err());
15499        let saved = RunState::load(&runner.state.id).unwrap();
15500        assert!(saved.candidates.is_empty());
15501        let event = saved
15502            .events
15503            .iter()
15504            .find(|e| e.node == "prep-intent")
15505            .unwrap();
15506        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15507        assert_eq!(intent["branch"], branch);
15508        assert_eq!(intent["start"], runner.state.base_commit);
15509        assert_eq!(
15510            intent["worktree"],
15511            runner
15512                .state
15513                .worktree_root()
15514                .join(format!("cand-{label}"))
15515                .to_string_lossy()
15516                .as_ref()
15517        );
15518        assert_eq!(intent["picks"], serde_json::json!([]));
15519    }
15520
15521    #[tokio::test]
15522    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15523        ask_test_home();
15524        let tmp = tempfile::tempdir().unwrap();
15525        let repo = tmp.path().join("repo");
15526        std::fs::create_dir_all(&repo).unwrap();
15527        init_repo(&repo);
15528        let mut runner = runner_at(RunStatus::Prep);
15529        runner.state.repo = repo.clone();
15530        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15531        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15532        runner.state.sources.pick_commits.push(refs::Input {
15533            token: "missing".into(),
15534            sha: "0".repeat(40),
15535            provenance: refs::Provenance::ExplicitCli,
15536        });
15537        assert!(runner.prep().await.is_err());
15538        assert!(runner.state.candidates.is_empty());
15539        assert!(!runner.state.worktree_root().exists());
15540    }
15541
15542    #[tokio::test]
15543    async fn prose_and_legacy_inference_never_import_into_candidates() {
15544        ask_test_home();
15545        let tmp = tempfile::tempdir().unwrap();
15546        let repo = tmp.path().join("repo");
15547        std::fs::create_dir_all(&repo).unwrap();
15548        init_repo(&repo);
15549        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15550        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15551            .await
15552            .unwrap();
15553        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15554        git::git(&repo, &["add", "."]).await.unwrap();
15555        git::git(&repo, &["commit", "-m", "historical implementation"])
15556            .await
15557            .unwrap();
15558        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15559        git::git(&repo, &["checkout", "main"]).await.unwrap();
15560        for instruction in [
15561            format!("Example: magi/history/A at {source}"),
15562            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15563            format!("Never import magi/history/A or {source}"),
15564            format!("Conversation answer: I mentioned {source} only as background"),
15565        ] {
15566            let mut runner = runner_at(RunStatus::Prep);
15567            runner.state.repo = repo.clone();
15568            runner.state.base_commit = base.clone();
15569            runner.state.instruction = instruction.clone();
15570            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15571            runner.state.config.blind.commit_msg_hook = false;
15572            runner.state.seeds = vec![refs::Seed {
15573                token: "magi/history/A".into(),
15574                sha: source.clone(),
15575                kind: refs::SeedKind::Unmerged,
15576                branch: true,
15577                detail: String::new(),
15578            }];
15579            runner.roles.implementers = vec![spec("one")];
15580            assert_eq!(seeded_instruction(&runner.state), instruction);
15581            // Model an old record with inferred seeds but no structured input field.
15582            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15583            legacy.as_object_mut().unwrap().remove("sources");
15584            runner.state = serde_json::from_value(legacy).unwrap();
15585            assert_eq!(runner.state.sources, refs::Sources::default());
15586            runner.state.save().unwrap();
15587            runner.state = RunState::load(&runner.state.id).unwrap();
15588            runner.prep().await.unwrap();
15589            for candidate in &runner.state.candidates {
15590                assert_eq!(
15591                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15592                    base
15593                );
15594                assert!(!candidate.worktree.join("unwanted").exists());
15595            }
15596        }
15597    }
15598
15599    #[tokio::test]
15600    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15601        ask_test_home();
15602        let tmp = tempfile::tempdir().unwrap();
15603        let repo = tmp.path().join("repo");
15604        std::fs::create_dir_all(&repo).unwrap();
15605        init_repo(&repo);
15606        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15607        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15608        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15609        git::git(&repo, &["add", "."]).await.unwrap();
15610        git::git(&repo, &["commit", "-m", "required seed"])
15611            .await
15612            .unwrap();
15613        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15614        git::git(&repo, &["checkout", "main"]).await.unwrap();
15615        let mut runner = runner_at(RunStatus::Prep);
15616        runner.state.repo = repo.clone();
15617        runner.state.base_commit = base.clone();
15618        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15619        runner.state.config.graph.advise = false;
15620        runner.state.config.blind.commit_msg_hook = false;
15621        runner.state.sources =
15622            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15623                .await
15624                .unwrap();
15625        runner.roles.implementers = vec![spec("one"), spec("two")];
15626        runner.roles.judges = vec![spec("judge")];
15627        let label = blind::assign_labels(2, runner.state.seed)[0];
15628        let branch = runner.state.branch_for(label);
15629        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15630        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15631            .await
15632            .unwrap();
15633        runner.state.candidates.push(Candidate {
15634            index: 0,
15635            label,
15636            agent: "one".into(),
15637            branch,
15638            worktree,
15639            summary: String::new(),
15640            stat: String::new(),
15641            files: 0,
15642            commits: 0,
15643            empty: false,
15644            failed: None,
15645            verified_noop: None,
15646            duration_ms: 0,
15647            folded: false,
15648        });
15649        runner.state.save().unwrap();
15650        // Reload the checkpoint immediately before its first cherry-pick.
15651        runner.state = RunState::load(&runner.state.id).unwrap();
15652        assert_eq!(runner.state.prep_complete, Some(false));
15653        runner.prep().await.unwrap();
15654        assert_eq!(runner.state.prep_complete, Some(true));
15655        assert_eq!(runner.state.candidates.len(), 2);
15656        for candidate in &runner.state.candidates {
15657            assert_eq!(
15658                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15659                git::tree_of(&repo, &seed).await.unwrap()
15660            );
15661            assert_eq!(
15662                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15663                "required commit"
15664            );
15665        }
15666        assert!(runner.state.worktree_root().join("judge-1").exists());
15667        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15668        runner.prep().await.unwrap();
15669        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15670
15671        runner.state.prep_complete = Some(false);
15672        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15673            run: Some(format!(
15674                "test -f '{}'",
15675                tmp.path().join("dependency-ready").display()
15676            )),
15677            ..Default::default()
15678        }];
15679        assert!(runner.prep().await.is_err());
15680        assert!(runner.state.prep_failure.is_some());
15681        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15682        runner.state = RunState::load(&runner.state.id).unwrap();
15683        assert!(
15684            runner.prep().await.is_err(),
15685            "correcting the cause alone must not auto-retry"
15686        );
15687        let failed_at = runner.state.events.last().unwrap().at;
15688        assert!(
15689            runner.retry_preparation(failed_at).is_err(),
15690            "old approvals cannot unlock the hold"
15691        );
15692        let mut task = crate::queue::Task::new(
15693            "retry".into(),
15694            "retry setup".into(),
15695            repo.clone(),
15696            crate::queue::Source::Human,
15697        );
15698        task.start(runner.state.id.clone());
15699        task.hold_machine(Some("setup failed".into()));
15700        task.release_operator("cli", None);
15701        runner
15702            .retry_preparation(task.resumption.as_ref().unwrap().at)
15703            .unwrap();
15704        runner.prep().await.unwrap();
15705        assert!(runner.state.prep_failure.is_none());
15706        assert_eq!(runner.state.prep_complete, Some(true));
15707        assert_eq!(runner.state.candidates.len(), 2);
15708        assert!(runner.state.worktree_root().join("judge-1").exists());
15709        for candidate in &runner.state.candidates {
15710            assert_eq!(
15711                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15712                git::tree_of(&repo, &seed).await.unwrap()
15713            );
15714        }
15715    }
15716
15717    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15718        let mut heads = Vec::new();
15719        for candidate in candidates {
15720            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15721        }
15722        heads
15723    }
15724
15725    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15726        ask_test_home();
15727        let tmp = tempfile::tempdir().unwrap();
15728        let repo = tmp.path().join("repo");
15729        std::fs::create_dir(&repo).unwrap();
15730        init_repo(&repo);
15731        let mut runner = runner_at(RunStatus::Reviewing);
15732        runner.state.repo = repo.clone();
15733        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15734        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15735        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15736            let branch = format!("candidate-{label}");
15737            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15738            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15739                .await
15740                .unwrap();
15741            if label == 'B' {
15742                std::fs::write(worktree.join("change"), "winner").unwrap();
15743                git::rescue_commit(&worktree, "candidate change")
15744                    .await
15745                    .unwrap();
15746            }
15747            runner.state.candidates.push(Candidate {
15748                index,
15749                label,
15750                agent: "alpha".into(),
15751                branch,
15752                worktree,
15753                summary: "completed reply".into(),
15754                stat: String::new(),
15755                files: usize::from(label == 'B'),
15756                commits: usize::from(label == 'B'),
15757                empty: label == 'A',
15758                failed: None,
15759                verified_noop: None,
15760                duration_ms: 1,
15761                folded: false,
15762            });
15763        }
15764        runner.state.judge_skipped = true;
15765        runner.state.tally = Some(Tally {
15766            first_choice: BTreeMap::new(),
15767            borda: BTreeMap::new(),
15768            winner: 'B',
15769            rankings: 0,
15770            unanimous_initial: false,
15771            deliberated: false,
15772            changed_votes: 0,
15773            unanimous_final: false,
15774            tie_break: None,
15775            judges: 0,
15776            present: 0,
15777            quorum: 0,
15778            met_quorum: true,
15779            uncontested: Some("only B produced a change".into()),
15780        });
15781        runner.state.save().unwrap();
15782        (tmp, runner)
15783    }
15784
15785    #[tokio::test]
15786    #[cfg(unix)]
15787    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15788        for (solo, failed_checkpoint) in
15789            [(true, false), (false, false), (true, true), (false, true)]
15790        {
15791            let (_tmp, mut runner) = preservation_resume_fixture().await;
15792            if solo {
15793                runner.state.candidates.retain(|c| c.label == 'A');
15794            }
15795            runner.state.candidates[0].empty = false;
15796            runner.state.candidates[0].summary.clear();
15797            runner.state.tally = None;
15798            runner.state.judge_skipped = false;
15799            runner.state.status = RunStatus::Implementing;
15800            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15801            let mut implementer = spec("alpha");
15802            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15803            runner.roles.implementers = vec![implementer];
15804            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15805            if failed_checkpoint {
15806                std::fs::create_dir_all(&patch).unwrap();
15807            }
15808            runner.state.save().unwrap();
15809            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15810            drop(pressure);
15811            runner.state = RunState::load(&runner.state.id).unwrap();
15812            assert_eq!(runner.state.status, RunStatus::Blocked);
15813            assert!(runner.state.interrupted_implementations.contains(&'A'));
15814            assert_eq!(runner.state.candidates[0].commits, 0);
15815            assert!(!runner.state.candidates[0].empty);
15816            assert!(runner.state.candidates[0].failed.is_none());
15817            if failed_checkpoint {
15818                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15819                std::fs::remove_dir(patch).unwrap();
15820            }
15821            runner.state.resource_block = None;
15822            runner.state.config.disk.min_free_bytes = 0;
15823            runner.roles.implementers[0].command = vec![
15824                "sh".into(),
15825                "-c".into(),
15826                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15827            ];
15828            runner.implement().await.unwrap();
15829            assert!(runner.state.interrupted_implementations.is_empty());
15830            assert!(runner.state.candidate_preservation.is_empty());
15831            assert_eq!(runner.state.candidates[0].commits, 1);
15832            assert!(!runner.state.candidates[0].empty);
15833            assert!(
15834                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15835                    .unwrap()
15836                    .contains("retried empty interrupted seat")
15837            );
15838            if !solo {
15839                assert_eq!(runner.state.candidates[1].commits, 1);
15840            }
15841        }
15842    }
15843
15844    #[tokio::test]
15845    #[cfg(unix)]
15846    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15847        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15848            let (_tmp, mut runner) = preservation_resume_fixture().await;
15849            runner.state.candidates.retain(|c| c.label == 'A');
15850            runner.state.candidates[0].empty = false;
15851            runner.state.candidates[0].summary.clear();
15852            runner.state.tally = None;
15853            runner.state.judge_skipped = false;
15854            runner.state.status = RunStatus::Implementing;
15855            // Force only the real post-spawn watchdog, without failing the
15856            // separate admission check. Existing edits predate its measurement.
15857            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15858            let wt = runner.state.candidates[0].worktree.clone();
15859            std::fs::write(
15860                wt.join("interrupted-source"),
15861                "retain implementation work\n",
15862            )
15863            .unwrap();
15864            if committed {
15865                git::rescue_commit(&wt, "implementation checkpoint")
15866                    .await
15867                    .unwrap();
15868            }
15869            let lock = git::git(
15870                &wt,
15871                &[
15872                    "rev-parse",
15873                    "--path-format=absolute",
15874                    "--git-path",
15875                    "index.lock",
15876                ],
15877            )
15878            .await
15879            .unwrap();
15880            if block_rescue {
15881                std::fs::write(&lock, "fixture-owned lock").unwrap();
15882            }
15883            let mut implementer = spec("alpha");
15884            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15885            runner.roles.implementers = vec![implementer];
15886            runner.state.save().unwrap();
15887            let result = runner.implement().await;
15888            drop(pressure);
15889            assert_eq!(result.is_err(), block_rescue);
15890            assert_eq!(
15891                runner.state.resource_block.as_ref().unwrap().kind,
15892                "disk pressure"
15893            );
15894            assert!(
15895                runner
15896                    .state
15897                    .events
15898                    .iter()
15899                    .any(|e| e.message.contains("agent execution interrupted"))
15900            );
15901            assert!(
15902                runner
15903                    .state
15904                    .dir()
15905                    .join("artifacts/candidate-A-recovery/staged.patch")
15906                    .exists()
15907            );
15908            if block_rescue {
15909                let saved = RunState::load(&runner.state.id).unwrap();
15910                assert!(saved.candidate_preservation.contains_key(&'A'));
15911                assert!(!saved.candidates[0].empty);
15912                assert_eq!(
15913                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15914                    "retain implementation work\n"
15915                );
15916                std::fs::remove_file(lock).unwrap();
15917                runner.state = saved;
15918            } else {
15919                assert_eq!(runner.state.status, RunStatus::Blocked);
15920                assert_eq!(runner.state.candidates[0].commits, 1);
15921            }
15922            // Recovery must save the candidate before deciding whether another
15923            // implementation is needed. No implementer is available on retry.
15924            runner.roles.implementers.clear();
15925            runner.state.config.disk.min_free_bytes = 0;
15926            runner.state.resource_block = None;
15927            runner.implement().await.unwrap();
15928            assert!(runner.state.candidate_preservation.is_empty());
15929            assert_eq!(runner.state.candidates[0].commits, 1);
15930            assert!(!runner.state.candidates[0].empty);
15931            assert!(git::is_clean(&wt).await.unwrap());
15932            assert!(
15933                crate::run::read_artifact(&runner.state, "cand-A.patch")
15934                    .unwrap()
15935                    .contains("retain implementation work")
15936            );
15937        }
15938    }
15939
15940    #[tokio::test]
15941    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15942        let (_tmp, mut runner) = preservation_resume_fixture().await;
15943        // Schema 15 already confirmed the unfolded no-change result.
15944        runner.implement().await.unwrap();
15945        assert!(runner.state.candidate_preservation.is_empty());
15946        assert!(
15947            !runner
15948                .state
15949                .dir()
15950                .join("artifacts/candidate-A-recovery")
15951                .exists()
15952        );
15953        runner.fold_losers().await.unwrap();
15954        assert!(!runner.state.candidates[0].worktree.exists());
15955        for schema in [14, crate::run::SCHEMA] {
15956            runner.state.schema = schema;
15957            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15958            runner.state.save().unwrap();
15959            runner.state = RunState::load(&runner.state.id).unwrap();
15960            runner.implement().await.unwrap();
15961            assert!(runner.state.candidate_preservation.is_empty());
15962            assert!(runner.state.candidates[0].folded);
15963            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15964        }
15965    }
15966
15967    #[tokio::test]
15968    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15969        let (_tmp, mut runner) = preservation_resume_fixture().await;
15970        runner.state.candidates.retain(|c| c.label == 'A');
15971        runner.state.tally = None;
15972        runner.state.judge_skipped = false;
15973        let candidate = &mut runner.state.candidates[0];
15974        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15975        candidate.failed = Some("deployment validation unavailable".into());
15976        std::fs::write(
15977            candidate.worktree.join("repair"),
15978            "future launcher policy\n",
15979        )
15980        .unwrap();
15981        runner
15982            .state
15983            .candidate_preservation
15984            .insert('A', "awaiting persistence".into());
15985        runner.persist_candidates().await.unwrap();
15986        let candidate = &runner.state.candidates[0];
15987        assert!(!candidate.empty);
15988        assert_eq!(candidate.commits, 1);
15989        assert_eq!(candidate.files, 1);
15990        assert!(candidate.failed.is_none());
15991        assert!(candidate.summary.contains("EPERM"));
15992        assert!(
15993            crate::run::read_artifact(&runner.state, "cand-A.patch")
15994                .unwrap()
15995                .contains("future launcher policy")
15996        );
15997        assert!(runner.state.candidate_preservation.is_empty());
15998    }
15999
16000    #[tokio::test]
16001    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16002        let (_tmp, mut runner) = preservation_resume_fixture().await;
16003        let wt = runner.state.candidates[0].worktree.clone();
16004        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16005        runner.state.schema = 14;
16006        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16007        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16008        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16009        for _ in 0..2 {
16010            assert!(runner.implement().await.is_err());
16011            assert_eq!(runner.state.status, RunStatus::Blocked);
16012            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16013            assert!(
16014                runner.state.candidates[0].empty,
16015                "old verdict must remain unusable"
16016            );
16017            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16018            assert!(runner.state.judge_skipped);
16019            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16020            assert!(
16021                crate::run::read_artifact(&runner.state, "cand-A.patch")
16022                    .unwrap()
16023                    .contains("owned edits")
16024            );
16025            assert!(wt.exists());
16026            assert_eq!(
16027                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16028                    .await
16029                    .unwrap(),
16030                1
16031            );
16032            runner.state = RunState::load(&runner.state.id).unwrap();
16033        }
16034    }
16035
16036    #[tokio::test]
16037    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16038        let (_tmp, mut runner) = preservation_resume_fixture().await;
16039        let wt = runner.state.candidates[0].worktree.clone();
16040        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16041        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16042        let index_lock = PathBuf::from(
16043            git::git(
16044                &wt,
16045                &[
16046                    "rev-parse",
16047                    "--path-format=absolute",
16048                    "--git-path",
16049                    "index.lock",
16050                ],
16051            )
16052            .await
16053            .unwrap(),
16054        );
16055        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16056        runner.state.schema = 14;
16057        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16058        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16059        assert!(runner.implement().await.is_err());
16060        assert!(runner.after_implement().is_err());
16061        assert!(runner.fold_losers().await.is_err());
16062        assert_eq!(
16063            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16064            "failed seat edits"
16065        );
16066        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16067        runner.state = RunState::load(&runner.state.id).unwrap();
16068        std::fs::remove_file(index_lock).unwrap();
16069        // Recovery finds changes that the old uncontested result never judged.
16070        assert!(runner.implement().await.is_err());
16071        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16072        assert!(runner.state.candidates[0].empty);
16073        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16074        assert!(
16075            crate::run::read_artifact(&runner.state, "cand-A.patch")
16076                .unwrap()
16077                .contains("failed seat edits")
16078        );
16079        assert!(wt.exists());
16080    }
16081
16082    #[tokio::test]
16083    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16084        let (_tmp, mut runner) = preservation_resume_fixture().await;
16085        let repo = runner.state.repo.clone();
16086        let common = git::git(
16087            &repo,
16088            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16089        )
16090        .await
16091        .unwrap();
16092        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16093        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16094        assert!(runner.fold_losers().await.is_err());
16095        assert!(!runner.state.candidates[0].worktree.exists());
16096        assert!(!runner.state.candidates[0].folded);
16097        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16098        runner.state = RunState::load(&runner.state.id).unwrap();
16099        std::fs::remove_file(branch_lock).unwrap();
16100        let judge = runner.state.worktree_root().join("judge-1");
16101        git::worktree_add_detached(&repo, &judge, "main")
16102            .await
16103            .unwrap();
16104        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16105        runner.roles.judges = vec![spec("judge")];
16106        assert!(runner.fold_losers().await.is_err());
16107        runner.state = RunState::load(&runner.state.id).unwrap();
16108        assert!(
16109            runner.state.candidates[0].folded,
16110            "candidate progress saved before judge failure"
16111        );
16112        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16113        assert!(judge.join("uncommitted").exists());
16114        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16115        runner.fold_losers().await.unwrap();
16116        assert!(!judge.exists());
16117        runner.fold_losers().await.unwrap();
16118    }
16119
16120    #[tokio::test]
16121    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16122        let (_tmp, mut runner) = preservation_resume_fixture().await;
16123        runner.state.candidates.retain(|c| c.label == 'B');
16124        runner.state.tally = None;
16125        runner.state.judge_skipped = false;
16126        runner.state.prep_complete = Some(true);
16127        runner.state.config.graph.advise = false;
16128        runner.state.config.graph.selected_implementer = Some("alpha".into());
16129        let wt = runner.state.candidates[0].worktree.clone();
16130        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16131        runner
16132            .state
16133            .candidate_preservation
16134            .insert('B', "awaiting persistence".into());
16135        let index_lock = git::git(
16136            &wt,
16137            &[
16138                "rev-parse",
16139                "--path-format=absolute",
16140                "--git-path",
16141                "index.lock",
16142            ],
16143        )
16144        .await
16145        .unwrap();
16146        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16147        runner.state.save().unwrap();
16148        assert!(runner.execute_graph().await.is_err());
16149        assert_eq!(runner.state.status, RunStatus::Blocked);
16150        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16151        std::fs::remove_file(index_lock).unwrap();
16152        // Park at the next graph boundary: this tests the real selected-seat
16153        // hold guard without dispatching reviewers or publishing anything.
16154        for _ in 0..2 {
16155            runner.state = RunState::load(&runner.state.id).unwrap();
16156            runner.pause.park();
16157            runner.execute_graph().await.unwrap();
16158            assert_eq!(runner.state.status, RunStatus::Implementing);
16159            assert!(runner.state.candidate_preservation.is_empty());
16160            assert_eq!(runner.state.candidates[0].commits, 2);
16161            assert!(runner.state.tally.is_none());
16162        }
16163        runner.implement().await.unwrap();
16164        assert_eq!(runner.state.status, RunStatus::Judging);
16165        // A genuine selected-implementer hold must still require release.
16166        runner.state.status = RunStatus::Blocked;
16167        runner.state.candidates[0].failed = Some("implementation failed".into());
16168        runner.state.save().unwrap();
16169        let error = runner.execute_graph().await.unwrap_err();
16170        assert!(
16171            error
16172                .to_string()
16173                .contains("selected implementation is held")
16174        );
16175    }
16176
16177    #[tokio::test]
16178    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16179        let (_tmp, mut runner) = preservation_resume_fixture().await;
16180        let repo = runner.state.repo.clone();
16181        let candidate = runner.state.candidates[0].clone();
16182        // Remove only the disposable fixture directory, leaving Git's linked
16183        // worktree registration and branch intact, as in a vanished checkout.
16184        let registered = git::worktree_holding(&repo, &candidate.branch)
16185            .await
16186            .unwrap()
16187            .unwrap();
16188        assert_eq!(
16189            registered.canonicalize().unwrap(),
16190            candidate.worktree.canonicalize().unwrap()
16191        );
16192        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16193        assert_eq!(
16194            git::worktree_holding(&repo, &candidate.branch)
16195                .await
16196                .unwrap(),
16197            Some(registered)
16198        );
16199        assert!(
16200            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16201            "the stale registration initially keeps the branch checked out"
16202        );
16203        runner.fold_losers().await.unwrap();
16204        assert!(runner.state.candidates[0].folded);
16205        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16206        assert_eq!(
16207            git::worktree_holding(&repo, &candidate.branch)
16208                .await
16209                .unwrap(),
16210            None
16211        );
16212        runner.state = RunState::load(&runner.state.id).unwrap();
16213        runner.fold_losers().await.unwrap();
16214    }
16215
16216    #[tokio::test]
16217    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16218        let (_tmp, mut runner) = preservation_resume_fixture().await;
16219        let repo = runner.state.repo.clone();
16220        let candidate = runner.state.candidates[0].clone();
16221        let winner = runner.state.candidates[1].clone();
16222        let registered = git::worktree_holding(&repo, &candidate.branch)
16223            .await
16224            .unwrap()
16225            .unwrap();
16226        assert_eq!(
16227            registered.canonicalize().unwrap(),
16228            candidate.worktree.canonicalize().unwrap()
16229        );
16230        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16231        assert_eq!(
16232            git::worktree_holding(&repo, &candidate.branch)
16233                .await
16234                .unwrap(),
16235            Some(registered)
16236        );
16237        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16238
16239        for _ in 0..2 {
16240            fold_run(&mut runner.state, false, &crate::run::home())
16241                .await
16242                .unwrap();
16243            runner.state = RunState::load(&runner.state.id).unwrap();
16244            assert!(runner.state.candidates[0].folded);
16245            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16246            assert_eq!(
16247                git::worktree_holding(&repo, &candidate.branch)
16248                    .await
16249                    .unwrap(),
16250                None
16251            );
16252            assert!(!runner.state.candidates[1].folded);
16253            assert!(winner.worktree.exists());
16254            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16255        }
16256    }
16257
16258    #[tokio::test]
16259    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16260        ask_test_home();
16261        let tmp = tempfile::tempdir().unwrap();
16262        let repo = tmp.path().join("repo");
16263        std::fs::create_dir(&repo).unwrap();
16264        init_repo(&repo);
16265        let mut runner = runner_at(RunStatus::Implementing);
16266        runner.state.repo = repo.clone();
16267        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16268        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16269        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16270            let branch = format!("candidate-{label}");
16271            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16272            git::worktree_add_branch(&repo, &wt, &branch, "main")
16273                .await
16274                .unwrap();
16275            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16276            git::git(&wt, &["add", "README.md"]).await.unwrap();
16277            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16278            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16279            runner.state.candidates.push(Candidate {
16280                index,
16281                label,
16282                agent: "alpha".to_owned(),
16283                branch,
16284                worktree: wt,
16285                summary: "completed implementation".to_owned(),
16286                stat: String::new(),
16287                files: 0,
16288                commits: 0,
16289                empty: false,
16290                failed: None,
16291                verified_noop: None,
16292                duration_ms: 42,
16293                folded: false,
16294            });
16295            runner
16296                .state
16297                .candidate_preservation
16298                .insert(label, "awaiting persistence".to_owned());
16299        }
16300        let wt = runner.state.candidates[0].worktree.clone();
16301        let index_lock = git::git(
16302            &wt,
16303            &[
16304                "rev-parse",
16305                "--path-format=absolute",
16306                "--git-path",
16307                "index.lock",
16308            ],
16309        )
16310        .await
16311        .unwrap();
16312        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16313        runner.state.save().unwrap();
16314        assert!(runner.persist_candidates().await.is_err());
16315        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16316        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16317        assert_eq!(
16318            runner.state.candidates[1].commits, 1,
16319            "other results are still collected"
16320        );
16321        assert!(
16322            !runner.state.candidates[0].empty,
16323            "failed rescue is not no change"
16324        );
16325        assert_eq!(
16326            runner.state.candidates[0].summary,
16327            "completed implementation"
16328        );
16329        assert!(runner.after_implement().is_err());
16330        let home = tmp.path().join("fold-home");
16331        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16332        assert!(wt.join("new-file").exists());
16333        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16334        assert!(recovery.join("staged.patch").exists());
16335        assert!(recovery.join("unstaged.patch").exists());
16336        assert_eq!(
16337            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16338            "untracked edits\n"
16339        );
16340        runner.state.begin_pass();
16341        runner.state.end_pass_parked(ParkCause::Upgrade {
16342            from: "1.0".into(),
16343            to: Some("1.1".into()),
16344        });
16345        runner.state.save().unwrap();
16346        runner.state = RunState::load(&runner.state.id).unwrap();
16347        assert!(matches!(
16348            runner.state.passes.last().unwrap().end,
16349            Some(crate::run::PassEnd::Parked {
16350                cause: ParkCause::Upgrade { .. }
16351            })
16352        ));
16353        std::fs::remove_file(index_lock).unwrap();
16354        // No implementer is configured: a persistence retry cannot dispatch a seat.
16355        runner.implement().await.unwrap();
16356        assert!(runner.state.candidate_preservation.is_empty());
16357        assert!(
16358            runner
16359                .state
16360                .candidates
16361                .iter()
16362                .all(|c| c.commits == 1 && !c.empty)
16363        );
16364        assert!(git::is_clean(&wt).await.unwrap());
16365        assert!(
16366            crate::run::read_artifact(&runner.state, "cand-A.patch")
16367                .unwrap()
16368                .contains("later tracked edits")
16369        );
16370    }
16371
16372    #[tokio::test]
16373    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16374        ask_test_home();
16375        let tmp = tempfile::tempdir().unwrap();
16376        let repo = tmp.path().join("repo");
16377        std::fs::create_dir(&repo).unwrap();
16378        init_repo(&repo);
16379        let wt = tmp.path().join("candidate");
16380        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16381            .await
16382            .unwrap();
16383        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16384        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16385        assert!(wt.join("uncommitted").exists());
16386        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16387        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16388        let mut runner = runner_at(RunStatus::Implementing);
16389        runner.state.repo = repo.clone();
16390        runner.state.base_commit = "missing-base".to_owned();
16391        runner.state.candidates.push(Candidate {
16392            index: 0,
16393            label: 'A',
16394            agent: "alpha".into(),
16395            branch: "candidate".into(),
16396            worktree: wt.clone(),
16397            summary: "completed".into(),
16398            stat: String::new(),
16399            files: 0,
16400            commits: 0,
16401            empty: false,
16402            failed: None,
16403            verified_noop: None,
16404            duration_ms: 1,
16405            folded: false,
16406        });
16407        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16408        assert!(
16409            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16410                .await
16411                .is_err()
16412        );
16413        assert!(!runner.state.candidates[0].folded);
16414        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16415        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16416        runner
16417            .state
16418            .candidate_preservation
16419            .insert('A', "pending".into());
16420        assert!(runner.persist_candidates().await.is_err());
16421        assert!(!runner.state.candidates[0].empty);
16422        assert!(!runner.state.candidates[0].folded);
16423        assert!(wt.exists());
16424        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16425        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16426        std::fs::create_dir_all(&patch_path).unwrap();
16427        assert!(
16428            runner.persist_candidates().await.is_err(),
16429            "artifact failure stays pending"
16430        );
16431        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16432        assert!(!runner.state.candidates[0].empty);
16433        std::fs::remove_dir(patch_path).unwrap();
16434        runner.persist_candidates().await.unwrap();
16435        assert!(
16436            runner.state.candidates[0].empty,
16437            "only successful inspection confirms no change"
16438        );
16439    }
16440
16441    fn init_repo(dir: &Path) {
16442        let run = |args: &[&str]| {
16443            let out = std::process::Command::new("git")
16444                .args(args)
16445                .current_dir(dir)
16446                .quiet()
16447                .output()
16448                .expect("spawn git");
16449            assert!(
16450                out.status.success(),
16451                "git {args:?} failed: {}",
16452                String::from_utf8_lossy(&out.stderr)
16453            );
16454        };
16455        run(&["init", "-b", "main"]);
16456        run(&["config", "user.name", "magi test"]);
16457        run(&["config", "user.email", "magi@example.com"]);
16458        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16459        run(&["add", "-A"]);
16460        run(&["commit", "-m", "init"]);
16461    }
16462
16463    // `settle_questions` is what closes the ghost the phone showed: a run's
16464    // seat asked something, the run then ended, and nothing was left to
16465    // abandon the question it left `open`. `HOME` is a process-wide
16466    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16467    // first time it runs in the binary — every test below still reaches the
16468    // same directory whichever call won, and each gets its own run id from
16469    // `RunState::new`, so they never collide there.
16470    fn ask_test_home() {
16471        crate::run::pin_test_home();
16472    }
16473
16474    #[test]
16475    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16476        ask_test_home();
16477        let mut runner = runner_at(RunStatus::Prep);
16478        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16479        // A blocked temporary output path makes the terminal save fail while
16480        // preserving the already published reservation.
16481        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16482        let error = guard
16483            .finish(
16484                &mut runner.state,
16485                Err(
16486                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16487                        .into(),
16488                ),
16489            )
16490            .unwrap_err();
16491        assert_eq!(
16492            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16493            std::io::ErrorKind::PermissionDenied
16494        );
16495        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16496        let message = format!("{error:#}");
16497        assert!(message.contains("original refusal"));
16498        assert!(!message.contains("could not save terminal"));
16499        assert_eq!(
16500            error.chain().count(),
16501            2,
16502            "only startup attribution wraps the original error"
16503        );
16504        let saved = RunState::load(&runner.state.id).unwrap();
16505        assert_eq!(saved.status, RunStatus::Blocked);
16506        assert!(!saved.startup.unwrap().complete);
16507    }
16508
16509    #[test]
16510    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16511        ask_test_home();
16512        let mut runner = runner_at(RunStatus::Prep);
16513        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16514        let id = runner.state.id.clone();
16515        drop(guard);
16516        let saved = RunState::load(&id).unwrap();
16517        assert_eq!(saved.status, RunStatus::Blocked);
16518        assert!(!saved.startup.as_ref().unwrap().complete);
16519        assert!(
16520            saved
16521                .startup
16522                .unwrap()
16523                .failure
16524                .unwrap()
16525                .contains("interrupted")
16526        );
16527        assert!(Runner::resume(&id).is_err());
16528    }
16529
16530    #[test]
16531    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16532        ask_test_home();
16533        let mut runner = runner_at(RunStatus::Prep);
16534        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16535        let error = guard
16536            .finish(
16537                &mut runner.state,
16538                Err(
16539                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16540                        .into(),
16541                ),
16542            )
16543            .unwrap_err();
16544        assert_eq!(
16545            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16546            std::io::ErrorKind::PermissionDenied
16547        );
16548        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16549        let mut task = crate::queue::Task::new(
16550            "review".into(),
16551            "criteria".into(),
16552            runner.state.repo.clone(),
16553            crate::queue::Source::Human,
16554        );
16555        task.record_startup_failure(failure.clone());
16556        task.record_startup_failure(failure.clone());
16557        assert_eq!(task.startup_failures.len(), 1);
16558        assert!(task.runs.is_empty());
16559        assert_eq!(task.attempts, 0);
16560    }
16561
16562    #[cfg(unix)]
16563    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16564        ask_test_home();
16565        let tmp = tempfile::tempdir().unwrap();
16566        let repo = tmp.path().join("repo");
16567        std::fs::create_dir(&repo).unwrap();
16568        init_repo(&repo);
16569        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16570        let origin = tmp.path().join("origin.git");
16571        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16572            .await
16573            .unwrap();
16574        git::git(
16575            &repo,
16576            &["remote", "add", "origin", origin.to_str().unwrap()],
16577        )
16578        .await
16579        .unwrap();
16580        git::git(&repo, &["checkout", "-b", "fix/retained"])
16581            .await
16582            .unwrap();
16583        std::fs::write(repo.join("retained"), "retained change").unwrap();
16584        git::git(&repo, &["add", "."]).await.unwrap();
16585        git::git(&repo, &["commit", "-m", "retained implementation"])
16586            .await
16587            .unwrap();
16588        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16589        git::git(&repo, &["checkout", "main"]).await.unwrap();
16590        std::fs::write(repo.join("base-change"), "new base").unwrap();
16591        git::git(&repo, &["add", "."]).await.unwrap();
16592        git::git(&repo, &["commit", "-m", "base moved"])
16593            .await
16594            .unwrap();
16595        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16596        git::git(&repo, &["checkout", "fix/retained"])
16597            .await
16598            .unwrap();
16599        let mut runner = runner_at(RunStatus::Voting);
16600        runner.state.repo = repo.clone();
16601        runner.state.base_commit = base;
16602        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16603        runner.state.config.disk.min_free_bytes = 0;
16604        runner.state.config.graph.review_rounds = 1;
16605        runner.state.config.verify.e2e = vec!["true".into()];
16606        runner.state.config.verify.gate = vec!["true".into()];
16607        runner.state.candidates = vec![Candidate {
16608            index: 0,
16609            label: 'A',
16610            agent: "fixture".into(),
16611            branch: "fix/retained".into(),
16612            worktree: repo.clone(),
16613            summary: String::new(),
16614            stat: String::new(),
16615            files: 1,
16616            commits: 1,
16617            empty: false,
16618            failed: None,
16619            verified_noop: None,
16620            duration_ms: 0,
16621            folded: false,
16622        }];
16623        runner.state.tally = Some(Tally {
16624            first_choice: BTreeMap::new(),
16625            borda: BTreeMap::new(),
16626            winner: 'A',
16627            rankings: 0,
16628            unanimous_initial: true,
16629            deliberated: false,
16630            changed_votes: 0,
16631            unanimous_final: true,
16632            tie_break: None,
16633            judges: 0,
16634            present: 0,
16635            quorum: 0,
16636            met_quorum: true,
16637            uncontested: Some("review-only".into()),
16638        });
16639        let mut task = crate::queue::Task::new(
16640            "fixture".into(),
16641            "fixture".into(),
16642            repo,
16643            crate::queue::Source::Human,
16644        );
16645        task.link_run(&runner.state.id);
16646        crate::queue::Queue::open().put(&mut task).unwrap();
16647        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16648        runner.state.base_sync = Some(BaseSync {
16649            tip: String::new(),
16650            behind: 1,
16651            attempts: BASE_SYNC_ROUNDS,
16652            conflict: None,
16653            already_in: None,
16654        });
16655        runner.sync_to_base().await.unwrap();
16656        assert_eq!(runner.state.status, RunStatus::Blocked);
16657        (tmp, runner, head, task.id)
16658    }
16659
16660    #[cfg(unix)]
16661    #[tokio::test]
16662    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16663        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16664        if std::env::var_os(CHILD).is_none() {
16665            let home = tempfile::tempdir().unwrap();
16666            let output = std::process::Command::new(std::env::current_exe().unwrap())
16667                .args([
16668                    "--exact",
16669                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16670                    "--nocapture",
16671                ])
16672                .env(CHILD, "1")
16673                .env("MAGI_HOME", home.path())
16674                .env("MAGI_WORKER_HOME", home.path())
16675                .env("MAGI_WORKER_ID", "abcdef")
16676                .output()
16677                .unwrap();
16678            assert!(
16679                output.status.success()
16680                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16681                "{}\n{}",
16682                String::from_utf8_lossy(&output.stdout),
16683                String::from_utf8_lossy(&output.stderr)
16684            );
16685            return;
16686        }
16687        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16688        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16689        let home = crate::run::home();
16690        let queue = crate::queue::Queue::open();
16691        let mut owner = queue.get(&task).unwrap();
16692        owner.release_operator("test-release", None);
16693        owner.start(runner.state.id.clone());
16694        queue.put(&mut owner).unwrap();
16695        runner.state.driver_exited = true;
16696        runner.state.task_instruction = Some(owner.instruction.clone());
16697        runner.state.save().unwrap();
16698        let _claim = queue.claim(&task).unwrap();
16699        let reserve = |id: &str| {
16700            let directory = home.join("workers/jobs").join(id);
16701            std::fs::create_dir_all(&directory).unwrap();
16702            let job = crate::worker::Job {
16703                id: id.into(),
16704                sequence: 1,
16705                kind: "task".into(),
16706                key: format!("task:{task}"),
16707                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16708                generation: "fixture-generation".into(),
16709                version: None,
16710                status: crate::worker::Status::Running,
16711                result: None,
16712                pid: Some(std::process::id()),
16713                created_at: 0,
16714                finished_at: None,
16715            };
16716            std::fs::write(
16717                directory.join("job.json"),
16718                serde_json::to_vec(&job).unwrap(),
16719            )
16720            .unwrap();
16721            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16722        };
16723        let _own = reserve("abcdef");
16724        let other = reserve("fedcba");
16725        assert!(
16726            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16727                .await
16728                .is_err()
16729        );
16730        assert!(
16731            RunState::load(&runner.state.id)
16732                .unwrap()
16733                .base_sync_recovery
16734                .authorizations
16735                .is_empty()
16736        );
16737        drop(other);
16738        assert!(crate::daemon::is_working_on(
16739            &home,
16740            &runner.state.id,
16741            Timestamp::now()
16742        ));
16743        assert!(
16744            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16745        );
16746        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16747            .await
16748            .unwrap();
16749        assert!(
16750            RunState::load(&runner.state.id)
16751                .unwrap()
16752                .base_recovery_pending()
16753        );
16754    }
16755
16756    #[cfg(unix)]
16757    #[tokio::test]
16758    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16759        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16760        let home = crate::run::home();
16761        runner.state.driver_pid = Some(std::process::id());
16762        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16763        runner.state.driver_exited = false;
16764        runner.state.begin_pass();
16765        runner.state.save().unwrap();
16766        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16767        driver.record(&runner.state).unwrap();
16768        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16769        // Preserve the durable claim as if the process died before Drop saved.
16770        drop(driver);
16771        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16772        assert_eq!(
16773            runner.state.liveness_under(&home, false),
16774            crate::run::Liveness::Dead
16775        );
16776        let queue = crate::queue::Queue::open();
16777        let _claim = queue.claim(&task).unwrap();
16778        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16779            .await
16780            .unwrap();
16781        let disk = RunState::load(&runner.state.id).unwrap();
16782        assert!(!disk.driver_exited);
16783        assert!(disk.base_recovery_pending());
16784    }
16785
16786    #[cfg(unix)]
16787    #[tokio::test]
16788    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16789        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16790        if std::env::var_os(CHILD).is_none() {
16791            use std::os::unix::fs::PermissionsExt;
16792            let tmp = tempfile::tempdir().unwrap();
16793            let gh = tmp.path().join("gh");
16794            std::fs::write(&gh, r#"#!/bin/sh
16795case "$1 $2" in
16796  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16797  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16798  *) exit 2 ;;
16799esac
16800"#).unwrap();
16801            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16802            let mut paths = vec![tmp.path().to_path_buf()];
16803            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16804            let output = std::process::Command::new(std::env::current_exe().unwrap())
16805                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16806                .env(CHILD, "1")
16807                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16808                .env("PATH", std::env::join_paths(paths).unwrap())
16809                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16810                .output().unwrap();
16811            assert!(
16812                output.status.success()
16813                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16814                "{}\n{}",
16815                String::from_utf8_lossy(&output.stdout),
16816                String::from_utf8_lossy(&output.stderr)
16817            );
16818            return;
16819        }
16820        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16821        let repo = runner.state.repo.clone();
16822        git::git(&repo, &["checkout", "main"]).await.unwrap();
16823        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16824        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16825        git::git(&repo, &["checkout", "fix/retained"])
16826            .await
16827            .unwrap();
16828        runner.state.pr = Some(crate::run::PrRecord {
16829            url: "https://example.test/project/pull/1".into(),
16830            number: 1,
16831            state: "open".into(),
16832            checks: "unknown".into(),
16833            round: 0,
16834            rounds: 1,
16835            red_at_merge: Vec::new(),
16836        });
16837        runner.sync_to_base().await.unwrap();
16838        assert_eq!(runner.state.status, RunStatus::Blocked);
16839        assert_eq!(
16840            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16841            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16842        );
16843        assert!(runner.state.base_sync_closure_pending());
16844        assert!(crate::handover::can_resume(&runner.state));
16845        let queue = crate::queue::Queue::open();
16846        assert!(!crate::base_sync_recovery::held_task(
16847            &queue,
16848            &queue.get(&task).unwrap()
16849        ));
16850        assert!(runner.state.authorize_base_sync(&task).is_err());
16851        // Legacy closure stops have no typed stop but must retain the checkpoint.
16852        runner.state.base_sync_recovery.stop = None;
16853        runner.state.status = RunStatus::Voting;
16854        runner.state.gate_ran = true;
16855        runner.state.config.graph.review_rounds = 0;
16856        runner.state.save().unwrap();
16857        assert!(crate::handover::can_resume(&runner.state));
16858        runner.state = RunState::load(&runner.state.id).unwrap();
16859        runner.execute_graph().await.unwrap();
16860        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16861        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16862        assert_eq!(
16863            runner.state.base_sync.as_ref().unwrap().attempts,
16864            BASE_SYNC_ROUNDS
16865        );
16866        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16867        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16868        assert!(runner.state.reviews.is_empty());
16869        assert!(runner.state.gate.is_empty());
16870    }
16871
16872    #[cfg(unix)]
16873    #[tokio::test]
16874    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16875        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16876        runner.state.driver_exited = true;
16877        runner.state.save().unwrap();
16878        let home = crate::run::home();
16879        let queue = crate::queue::Queue::open();
16880        let _claim = queue.claim(&task).unwrap();
16881        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16882        assert!(
16883            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16884                .await
16885                .is_err()
16886        );
16887        drop(lease);
16888        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16889        assert!(
16890            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16891                .await
16892                .is_err()
16893        );
16894        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16895        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16896            .await
16897            .unwrap();
16898        let disk = RunState::load(&runner.state.id).unwrap();
16899        assert!(disk.base_recovery_pending());
16900        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16901        assert!(
16902            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16903                .await
16904                .is_err()
16905        );
16906        assert_eq!(
16907            RunState::load(&runner.state.id)
16908                .unwrap()
16909                .base_sync_recovery
16910                .authorizations
16911                .len(),
16912            1
16913        );
16914        let mut owner = queue.get(&task).unwrap();
16915        owner.runs.push("20260101-000000-ffff".into());
16916        queue.put(&mut owner).unwrap();
16917        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16918            .await
16919            .unwrap_err();
16920        assert!(error.to_string().contains("run task ownership changed"));
16921    }
16922
16923    #[cfg(unix)]
16924    #[tokio::test]
16925    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16926        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16927        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16928        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16929        legacy["status"] = serde_json::json!("voting");
16930        runner.state = serde_json::from_value(legacy).unwrap();
16931        runner.state.save().unwrap();
16932        for _ in 0..3 {
16933            runner.execute_graph().await.unwrap();
16934            assert_eq!(runner.state.status, RunStatus::Blocked);
16935            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16936            assert!(!crate::handover::can_resume(&runner.state));
16937            assert_eq!(
16938                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16939                head
16940            );
16941            assert!(runner.state.reviews.is_empty());
16942            assert!(!runner.state.gate_ran);
16943        }
16944        let notice = crate::notices::run_ended(&runner.state).unwrap();
16945        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16946        runner.state.begin_pass();
16947        runner.state.status = RunStatus::Voting;
16948        runner.deliberate().await.unwrap();
16949        runner.vote().await.unwrap();
16950        runner.finish_after_tally().await.unwrap();
16951        assert_eq!(runner.state.status, RunStatus::Blocked);
16952        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16953        assert_eq!(
16954            crate::notices::run_ended(&runner.state).unwrap().since,
16955            notice.since
16956        );
16957    }
16958
16959    #[cfg(unix)]
16960    #[tokio::test]
16961    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16962        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16963        for detail in [
16964            "fix/retained conflicts with origin/main and did not rebase: conflict",
16965            "rebased fix/retained locally but origin refused the push",
16966            "unknown failure",
16967        ] {
16968            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16969            runner.state.record_base_sync_stop(&head);
16970            assert!(runner.state.authorize_base_sync(&task).is_err());
16971            runner.state.status = RunStatus::Voting;
16972            runner.execute_graph().await.unwrap();
16973            assert_eq!(runner.state.status, RunStatus::Blocked);
16974            assert!(!crate::handover::can_resume(&runner.state));
16975            assert_eq!(
16976                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16977                head
16978            );
16979        }
16980    }
16981
16982    #[cfg(unix)]
16983    #[tokio::test]
16984    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
16985        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16986        runner.state.authorize_base_sync(&task).unwrap();
16987        assert!(runner.state.consume_base_recovery(&head));
16988        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
16989        runner.state.reserve_base_operation(&head, &tip);
16990        runner.state.save().unwrap();
16991        runner.state = RunState::load(&runner.state.id).unwrap();
16992        assert!(!runner.state.consume_base_recovery(&head));
16993        runner.sync_to_base().await.unwrap();
16994        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
16995        assert_eq!(runner.state.base_episode_used(), 1);
16996        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
16997        assert!(
16998            runner
16999                .state
17000                .base_sync_recovery
17001                .operations
17002                .last()
17003                .unwrap()
17004                .finished
17005        );
17006        assert!(!runner.state.base_sync_blocked());
17007        runner.sync_to_base().await.unwrap();
17008        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17009    }
17010
17011    #[cfg(unix)]
17012    #[tokio::test]
17013    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17014        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17015        // Both historical diagnostic spellings retain an exhaustion identity.
17016        let original = runner
17017            .state
17018            .base_sync
17019            .as_ref()
17020            .unwrap()
17021            .conflict
17022            .clone()
17023            .unwrap();
17024        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17025            original
17026                .replace("synchronization(s)", "rebase(s)")
17027                .replace("synchronizing", "rebasing"),
17028        );
17029        runner.state.base_sync_recovery = Default::default();
17030        runner.state.record_base_sync_stop(&head);
17031        assert_eq!(
17032            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17033            crate::base_sync_recovery::StopKind::Exhausted
17034        );
17035        runner.state.authorize_base_sync(&task).unwrap();
17036        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17037        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17038            .as_object_mut()
17039            .unwrap();
17040        auth.remove("interval");
17041        auth.remove("scope");
17042        runner.state = serde_json::from_value(legacy).unwrap();
17043        assert!(runner.state.consume_base_recovery(&head));
17044        assert_eq!(runner.state.base_episode_used(), 3);
17045        runner.sync_to_base().await.unwrap();
17046        assert_eq!(runner.state.base_episode_used(), 4);
17047        runner.state.resource_block = Some(crate::run::ResourceBlock {
17048            node: "gate".into(),
17049            kind: "cache".into(),
17050            detail: "legacy wait".into(),
17051            at: Timestamp::now(),
17052            cause: crate::run::ResourceBlockCause::Unrecorded,
17053        });
17054        runner.state.save().unwrap();
17055        assert!(
17056            runner
17057                .state
17058                .base_sync_recovery
17059                .episodes
17060                .last()
17061                .unwrap()
17062                .boundary
17063                .is_none()
17064        );
17065        runner.state.resource_block = None;
17066        advance_base_fixture(&runner).await;
17067        runner.sync_to_base().await.unwrap();
17068        assert!(runner.state.base_sync_blocked());
17069        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17070    }
17071
17072    #[cfg(unix)]
17073    async fn advance_base_fixture(runner: &Runner) {
17074        let repo = &runner.state.repo;
17075        for args in [
17076            vec!["checkout", "main"],
17077            vec!["commit", "--allow-empty", "-m", "base advanced"],
17078            vec!["push", "origin", "main"],
17079            vec!["checkout", "fix/retained"],
17080        ] {
17081            git::git(repo, &args).await.unwrap();
17082        }
17083    }
17084
17085    #[cfg(unix)]
17086    fn fresh_base_interval_fixture(runner: &mut Runner) {
17087        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17088        runner.state.base_sync.as_mut().unwrap().conflict = None;
17089        runner.state.base_sync_recovery = Default::default();
17090        runner.state.status = RunStatus::Voting;
17091    }
17092
17093    #[cfg(unix)]
17094    #[tokio::test]
17095    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17096        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17097        fresh_base_interval_fixture(&mut runner);
17098        runner.state.begin_pass();
17099        runner.sync_to_base().await.unwrap();
17100        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17101        runner.state.resource_block = Some(crate::run::ResourceBlock {
17102            node: "gate".into(),
17103            kind: "build cache".into(),
17104            detail: "busy".into(),
17105            at: Timestamp::now(),
17106            cause: crate::run::ResourceBlockCause::Timeout {
17107                waited_secs: 1,
17108                limit_secs: 1,
17109            },
17110        });
17111        runner.state.save().unwrap();
17112        let queue = crate::queue::Queue::open();
17113        let mut owner = queue.get(&task).unwrap();
17114        runner.state.task_instruction = Some(owner.instruction.clone());
17115        owner.release_operator("fixture-owner", None);
17116        queue.put(&mut owner).unwrap();
17117        let approval = owner.resumption.clone().unwrap();
17118        runner.state.resource_block = None;
17119        runner.state.save().unwrap();
17120        runner.state = RunState::load(&runner.state.id).unwrap();
17121        advance_base_fixture(&runner).await;
17122        runner.sync_to_base().await.unwrap();
17123        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17124        assert_eq!(runner.state.base_episode_used(), 1);
17125        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17126        assert!(
17127            runner.state.base_sync_recovery.episodes[0]
17128                .boundary
17129                .as_ref()
17130                .unwrap()
17131                .consumed
17132        );
17133        assert!(
17134            !runner
17135                .state
17136                .renew_base_episode(&task, &approval, &head, "base")
17137        );
17138        for _ in 0..3 {
17139            owner.release_operator("fixture-owner", None);
17140            queue.put(&mut owner).unwrap();
17141            runner.state = RunState::load(&runner.state.id).unwrap();
17142            runner.state.begin_pass();
17143            runner.sync_to_base().await.unwrap();
17144            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17145            assert_eq!(runner.state.base_episode_used(), 1);
17146        }
17147        runner.state.resource_block = Some(crate::run::ResourceBlock {
17148            node: "gate".into(),
17149            kind: "disk pressure".into(),
17150            detail: "free space required".into(),
17151            at: Timestamp::now(),
17152            cause: crate::run::ResourceBlockCause::Other,
17153        });
17154        runner.state.end_pass_parked(ParkCause::Interrupt {
17155            reason: "disk pressure".into(),
17156        });
17157        runner.state.save().unwrap();
17158        owner.release_operator("fixture-owner-after-disk", None);
17159        queue.put(&mut owner).unwrap();
17160        runner.state.resource_block = None;
17161        advance_base_fixture(&runner).await;
17162        runner.sync_to_base().await.unwrap();
17163        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17164        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17165        assert_eq!(runner.state.base_episode_used(), 1);
17166        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17167    }
17168
17169    #[cfg(unix)]
17170    #[tokio::test]
17171    async fn base_recovery_continuous_advancement_stops_at_four() {
17172        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17173        fresh_base_interval_fixture(&mut runner);
17174        for spent in 1..=BASE_SYNC_ROUNDS {
17175            runner.sync_to_base().await.unwrap();
17176            assert!(!runner.state.base_sync_blocked());
17177            assert_eq!(runner.state.base_episode_used(), spent);
17178            advance_base_fixture(&runner).await;
17179        }
17180        runner.sync_to_base().await.unwrap();
17181        assert_eq!(
17182            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17183            crate::base_sync_recovery::StopKind::Exhausted
17184        );
17185        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17186        assert!(stop.unverified_sync);
17187        let action = runner.state.base_sync_next_action();
17188        assert!(action.contains("never reached its gate"));
17189        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17190        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17191        runner.state = RunState::load(&runner.state.id).unwrap();
17192        runner.sync_to_base().await.unwrap();
17193        assert_eq!(runner.state.base_episode_used(), 4);
17194        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17195    }
17196
17197    #[cfg(unix)]
17198    #[tokio::test]
17199    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17200        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17201        runner.state.ensure_base_episode();
17202        // Verification progress alone is not evidence of a successful sync.
17203        runner.state.record_base_progress(&head);
17204        runner.state.base_sync_recovery.stop = None;
17205        runner.state.record_base_sync_stop(&head);
17206        assert!(
17207            !runner
17208                .state
17209                .base_sync_recovery
17210                .stop
17211                .as_ref()
17212                .unwrap()
17213                .unverified_sync
17214        );
17215
17216        runner.state.finish_base_operation(Some(&head));
17217        // A gate for another head cannot verify this synchronization.
17218        runner.state.gate_ran = true;
17219        runner.state.gate_head = Some("older-head".into());
17220        runner.state.base_sync_recovery.stop = None;
17221        runner.state.record_base_sync_stop(&head);
17222        runner.state.save().unwrap();
17223        runner.state = RunState::load(&runner.state.id).unwrap();
17224        assert!(
17225            runner
17226                .state
17227                .base_sync_recovery
17228                .stop
17229                .as_ref()
17230                .unwrap()
17231                .unverified_sync
17232        );
17233
17234        runner.state.gate_head = Some(head.clone());
17235        runner.state.base_sync_recovery.stop = None;
17236        runner.state.record_base_sync_stop(&head);
17237        assert!(
17238            !runner
17239                .state
17240                .base_sync_recovery
17241                .stop
17242                .as_ref()
17243                .unwrap()
17244                .unverified_sync
17245        );
17246        assert!(
17247            !runner
17248                .state
17249                .base_sync_next_action()
17250                .contains("never reached its gate")
17251        );
17252
17253        // Historical records cannot establish the missing synchronization evidence.
17254        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17255        legacy["base_sync_recovery"]["stop"]
17256            .as_object_mut()
17257            .unwrap()
17258            .remove("unverified_sync");
17259        legacy["base_sync_recovery"]["episodes"][0]
17260            .as_object_mut()
17261            .unwrap()
17262            .remove("last_sync_head");
17263        runner.state = serde_json::from_value(legacy).unwrap();
17264        assert!(
17265            !runner
17266                .state
17267                .base_sync_recovery
17268                .stop
17269                .as_ref()
17270                .unwrap()
17271                .unverified_sync
17272        );
17273        assert!(
17274            runner.state.base_sync_recovery.episodes[0]
17275                .last_sync_head
17276                .is_none()
17277        );
17278    }
17279
17280    #[cfg(unix)]
17281    #[tokio::test]
17282    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17283        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17284        runner.state.authorize_base_sync(&task).unwrap();
17285        assert!(
17286            runner
17287                .state
17288                .base_sync_next_action()
17289                .contains("Approval recorded")
17290        );
17291        assert!(
17292            runner
17293                .state
17294                .base_sync_next_action()
17295                .contains("4/4 in episode 0, 4 lifetime")
17296        );
17297        assert!(runner.state.consume_base_recovery(&head));
17298        assert!(
17299            runner
17300                .state
17301                .base_sync_next_action()
17302                .contains("0/4 in episode 1, 4 lifetime")
17303        );
17304    }
17305
17306    #[cfg(unix)]
17307    #[tokio::test]
17308    async fn base_recovery_changed_head_invalidates_without_spending() {
17309        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17310        runner.state.authorize_base_sync(&task).unwrap();
17311        git::git(
17312            &runner.state.repo,
17313            &["commit", "--allow-empty", "-m", "external head change"],
17314        )
17315        .await
17316        .unwrap();
17317        runner.sync_to_base().await.unwrap();
17318        let a = runner
17319            .state
17320            .base_sync_recovery
17321            .authorizations
17322            .last()
17323            .unwrap();
17324        assert!(a.consumed);
17325        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17326        runner.state = RunState::load(&runner.state.id).unwrap();
17327        assert!(
17328            runner
17329                .state
17330                .base_sync_next_action()
17331                .contains("Approval invalidated")
17332        );
17333        assert!(!runner.state.base_recovery_pending());
17334        assert!(runner.state.base_sync_blocked());
17335        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17336    }
17337
17338    #[cfg(unix)]
17339    #[tokio::test]
17340    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17341        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17342        runner.state.driver_exited = true;
17343        runner.state.save().unwrap();
17344        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17345            .await
17346            .unwrap();
17347        runner.state = RunState::load(&runner.state.id).unwrap();
17348        advance_base_fixture(&runner).await;
17349        runner.sync_to_base().await.unwrap();
17350        let a = runner
17351            .state
17352            .base_sync_recovery
17353            .authorizations
17354            .last()
17355            .unwrap();
17356        assert!(
17357            a.consumed
17358                && a.invalidated
17359                    .as_deref()
17360                    .unwrap()
17361                    .contains("fetched base changed")
17362        );
17363        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17364        assert_eq!(
17365            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17366            head
17367        );
17368        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17369            .await
17370            .unwrap();
17371        runner.state = RunState::load(&runner.state.id).unwrap();
17372        runner.sync_to_base().await.unwrap();
17373        assert!(!runner.state.base_sync_blocked());
17374        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17375    }
17376
17377    #[cfg(unix)]
17378    #[tokio::test]
17379    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17380        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17381        runner.state.authorize_base_sync(&task).unwrap();
17382        assert!(runner.state.consume_base_recovery(&head));
17383        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17384        runner.state.reserve_base_operation(&head, &tip);
17385        runner.state.save().unwrap();
17386        let scratch = runner.state.dir().join("base-sync");
17387        assert_eq!(
17388            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17389                .await
17390                .unwrap(),
17391            crate::rebase::Rebased::Applied
17392        );
17393        runner.state = RunState::load(&runner.state.id).unwrap();
17394        runner.sync_to_base().await.unwrap();
17395        assert!(
17396            runner
17397                .state
17398                .base_sync_recovery
17399                .operations
17400                .last()
17401                .unwrap()
17402                .finished
17403        );
17404        assert_eq!(runner.state.base_episode_used(), 1);
17405        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17406        assert!(!runner.state.base_sync_blocked());
17407    }
17408
17409    #[cfg(unix)]
17410    #[tokio::test]
17411    async fn base_recovery_crash_preserves_untracked_files() {
17412        for ignored in [false, true] {
17413            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17414            runner.state.authorize_base_sync(&task).unwrap();
17415            assert!(runner.state.consume_base_recovery(&head));
17416            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17417            runner.state.reserve_base_operation(&head, &tip);
17418            runner.state.save().unwrap();
17419            let scratch = runner.state.dir().join("base-sync");
17420            assert_eq!(
17421                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17422                    .await
17423                    .unwrap(),
17424                crate::rebase::Rebased::Applied
17425            );
17426            let repo = &runner.state.repo;
17427            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17428            assert_ne!(result, head);
17429            if ignored {
17430                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17431                    .await
17432                    .unwrap();
17433                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17434            }
17435            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17436            assert!(
17437                git::git_raw(repo, &["diff", "--quiet", &head])
17438                    .await
17439                    .unwrap()
17440                    .ok()
17441            );
17442            // Resume after the branch moved, before its checkout was reset.
17443            runner.state = RunState::load(&runner.state.id).unwrap();
17444            runner.sync_to_base().await.unwrap();
17445            assert!(runner.state.base_sync_blocked());
17446            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17447            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17448            assert_eq!(
17449                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17450                result
17451            );
17452            assert_eq!(
17453                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17454                "retained work\n"
17455            );
17456            assert!(
17457                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17458                    .await
17459                    .unwrap()
17460                    .ok()
17461            );
17462        }
17463    }
17464
17465    #[cfg(unix)]
17466    #[tokio::test]
17467    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17468        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17469        runner.state.authorize_base_sync(&task).unwrap();
17470        assert!(runner.state.consume_base_recovery(&head));
17471        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17472        runner.state.reserve_base_operation(&head, &tip);
17473        runner.state.save().unwrap();
17474        let scratch = runner.state.dir().join("base-sync");
17475        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17476            .await
17477            .unwrap();
17478        git::sync_to_head(&runner.state.repo).await.unwrap();
17479        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17480        git::git(&runner.state.repo, &["add", "foreign"])
17481            .await
17482            .unwrap();
17483        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17484            .await
17485            .unwrap();
17486        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17487        runner.state = RunState::load(&runner.state.id).unwrap();
17488        runner.sync_to_base().await.unwrap();
17489        assert!(runner.state.base_sync_blocked());
17490        assert!(
17491            !runner
17492                .state
17493                .base_sync_recovery
17494                .operations
17495                .last()
17496                .unwrap()
17497                .finished
17498        );
17499        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17500        assert_eq!(
17501            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17502            foreign_head
17503        );
17504    }
17505
17506    #[cfg(unix)]
17507    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17508    {
17509        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17510        fresh_base_interval_fixture(&mut runner);
17511        let repo = &runner.state.repo;
17512        git::git(repo, &["checkout", "main"]).await.unwrap();
17513        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17514        git::git(repo, &["add", "retained"]).await.unwrap();
17515        git::git(repo, &["commit", "-m", "upstream retained change"])
17516            .await
17517            .unwrap();
17518        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17519        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17520        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17521        let scratch = runner.state.dir().join("base-sync");
17522        assert!(matches!(
17523            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17524                .await
17525                .unwrap(),
17526            git::RebaseStart::Conflicted(_)
17527        ));
17528        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17529            agent: "fixture".into(),
17530            paths: vec!["retained".into()],
17531            from: Some(head.clone()),
17532            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17533            index_outside: None,
17534            finished: false,
17535            error: None,
17536            launcher: None,
17537            infra: None,
17538            outage: None,
17539        });
17540        (tmp, runner, head, tip, scratch)
17541    }
17542
17543    #[cfg(unix)]
17544    #[tokio::test]
17545    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17546        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17547        // A resource stop from before reservations existed, with resolved
17548        // contents left for the launcher's ordinary staging continuation.
17549        std::fs::write(
17550            scratch.join("retained"),
17551            "upstream and retained resolution\n",
17552        )
17553        .unwrap();
17554        runner.state.resource_block = Some(crate::run::ResourceBlock {
17555            node: "rebase".into(),
17556            kind: "repository coordination".into(),
17557            detail: "busy".into(),
17558            at: Timestamp::now(),
17559            cause: crate::run::ResourceBlockCause::Other,
17560        });
17561        runner.state.save().unwrap();
17562        // Advance the remote without moving either checkout participating
17563        // in the standing rebase.
17564        let newer = git::git(
17565            &runner.state.repo,
17566            &[
17567                "commit-tree",
17568                "main^{tree}",
17569                "-p",
17570                "main",
17571                "-m",
17572                "base advanced during wait",
17573            ],
17574        )
17575        .await
17576        .unwrap();
17577        git::git(
17578            &runner.state.repo,
17579            &[
17580                "push",
17581                "origin",
17582                &format!("{}:refs/heads/main", newer.trim()),
17583            ],
17584        )
17585        .await
17586        .unwrap();
17587        runner.state = RunState::load(&runner.state.id).unwrap();
17588        runner.state.resource_block = None;
17589        runner.sync_to_base().await.unwrap();
17590        assert!(!runner.state.base_sync_blocked());
17591        let op = &runner.state.base_sync_recovery.operations[0];
17592        assert_eq!(op.head, head);
17593        assert_eq!(op.tip, tip);
17594        assert!(op.finished);
17595        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17596        assert_eq!(runner.state.rebase_fixes.len(), 1);
17597        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17598            .await
17599            .unwrap();
17600        assert_ne!(latest, tip);
17601        runner.state = RunState::load(&runner.state.id).unwrap();
17602        runner.sync_to_base().await.unwrap();
17603        assert!(!runner.state.base_sync_blocked());
17604        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17605        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17606        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17607    }
17608
17609    #[cfg(unix)]
17610    #[tokio::test]
17611    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17612        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17613        runner.state.reserve_base_operation(&head, &tip);
17614        runner.state.save().unwrap();
17615        std::fs::write(
17616            scratch.join("retained"),
17617            "upstream and retained resolution\n",
17618        )
17619        .unwrap();
17620        git::git(&scratch, &["add", "retained"]).await.unwrap();
17621        git::git(
17622            &scratch,
17623            &["-c", "core.editor=true", "rebase", "--continue"],
17624        )
17625        .await
17626        .unwrap();
17627        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17628        assert!(
17629            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17630                .await
17631                .unwrap()
17632                .is_empty()
17633        );
17634        assert_eq!(
17635            crate::rebase::finish(
17636                &mut runner.state,
17637                &scratch,
17638                "fix/retained",
17639                &head,
17640                &tip,
17641                &["retained".into()],
17642                ""
17643            )
17644            .await
17645            .unwrap(),
17646            crate::rebase::Rebased::Applied
17647        );
17648        assert!(!scratch.exists());
17649        // Simulate losing the caller before finish_base_operation/save.
17650        runner.state = RunState::load(&runner.state.id).unwrap();
17651        let op = &runner.state.base_sync_recovery.operations[0];
17652        assert!(!op.finished);
17653        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17654        runner.sync_to_base().await.unwrap();
17655        assert!(!runner.state.base_sync_blocked());
17656        assert!(runner.state.base_sync_recovery.operations[0].finished);
17657        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17658        assert_eq!(runner.state.rebase_fixes.len(), 1);
17659        let winner = runner.state.winner().unwrap().clone();
17660        let wait = runner.repository_wait();
17661        assert!(
17662            verify_current_head(&mut runner.state, &winner, &wait)
17663                .await
17664                .unwrap()
17665        );
17666        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17667    }
17668
17669    #[cfg(unix)]
17670    #[tokio::test]
17671    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17672        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17673            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17674            runner.state.reserve_base_operation(&head, &tip);
17675            runner.state.save().unwrap();
17676            std::fs::write(
17677                scratch.join("retained"),
17678                "upstream and retained resolution\n",
17679            )
17680            .unwrap();
17681            git::git(&scratch, &["add", "retained"]).await.unwrap();
17682            git::git(
17683                &scratch,
17684                &["-c", "core.editor=true", "rebase", "--continue"],
17685            )
17686            .await
17687            .unwrap();
17688            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17689            assert!(!git::rebase_in_progress(&scratch).await);
17690            assert!(
17691                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17692                    .await
17693                    .unwrap()
17694                    .is_empty()
17695            );
17696            match damage {
17697                Some("record") => {
17698                    let path =
17699                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17700                            .await
17701                            .unwrap();
17702                    std::fs::write(scratch.join(path), "").unwrap();
17703                }
17704                Some("foreign") => {
17705                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17706                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17707                    git::git(&scratch, &["commit", "-m", "foreign work"])
17708                        .await
17709                        .unwrap();
17710                }
17711                Some("dirty") => {
17712                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17713                }
17714                _ => {}
17715            }
17716            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17717            // Crash immediately after Git continuation, before finish runs.
17718            runner.state = RunState::load(&runner.state.id).unwrap();
17719            assert!(
17720                runner.state.base_sync_recovery.operations[0]
17721                    .validated_head
17722                    .is_none()
17723            );
17724            runner.sync_to_base().await.unwrap();
17725            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17726            assert_eq!(runner.state.base_episode_used(), 1);
17727            assert_eq!(runner.state.rebase_fixes.len(), 1);
17728            assert_eq!(
17729                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17730                retained_head
17731            );
17732            if damage.is_some() {
17733                assert!(runner.state.base_sync_blocked());
17734                assert!(scratch.exists());
17735                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17736                assert!(
17737                    runner.state.base_sync_recovery.operations[0]
17738                        .validated_head
17739                        .is_none()
17740                );
17741            } else {
17742                assert!(!runner.state.base_sync_blocked());
17743                assert!(!scratch.exists());
17744                assert!(runner.state.base_sync_recovery.operations[0].finished);
17745                assert_eq!(
17746                    runner.state.base_sync_recovery.operations[0]
17747                        .validated_head
17748                        .as_deref(),
17749                    Some(result.as_str())
17750                );
17751                let winner = runner.state.winner().unwrap().clone();
17752                let wait = runner.repository_wait();
17753                assert!(
17754                    verify_current_head(&mut runner.state, &winner, &wait)
17755                        .await
17756                        .unwrap()
17757                );
17758                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17759                runner.state = RunState::load(&runner.state.id).unwrap();
17760                runner.sync_to_base().await.unwrap();
17761                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17762            }
17763        }
17764    }
17765
17766    #[cfg(unix)]
17767    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17768        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17769        // This is an ordinary gate stop, not an authorized base-budget recovery.
17770        runner.state.prep_complete = Some(true);
17771        runner.state.base_sync = None;
17772        runner.state.base_sync_recovery = Default::default();
17773        runner.state.config.graph.gate_fix_rounds = 1;
17774        let mut review = review_round(true, 0, 1, 1, false, true);
17775        review.head = head.clone();
17776        review.verified_head = Some(head.clone());
17777        runner.state.reviews = vec![review];
17778        runner.state.gate = vec![CommandOutcome {
17779            command: "old gate".into(),
17780            code: Some(1),
17781            output_tail: "old failure".into(),
17782            duration_ms: 0,
17783            resource_blocked: false,
17784        }];
17785        runner.state.gate_head = Some(head.clone());
17786        runner.state.gate_ran = true;
17787        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17788            agent: "fixture".into(),
17789            failed: runner.state.gate.clone(),
17790            notes: "spent".into(),
17791            committed: false,
17792            error: None,
17793        }];
17794        runner.state.land_approval = Some(crate::run::LandApproval {
17795            question: "old approval".into(),
17796            head: head.clone(),
17797        });
17798        runner.state.status = RunStatus::Blocked;
17799        (tmp, runner, head)
17800    }
17801
17802    #[cfg(unix)]
17803    #[tokio::test]
17804    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17805        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17806            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17807            let phases = tmp.path().join("phases");
17808            runner.state.config.verify.e2e = vec![format!(
17809                "echo e2e >> '{}'; {}",
17810                phases.display(),
17811                if e2e_ok {
17812                    "true"
17813                } else {
17814                    "echo new-e2e-failure; false"
17815                }
17816            )];
17817            runner.state.config.verify.gate = vec![format!(
17818                "echo gate >> '{}'; {}",
17819                phases.display(),
17820                if gate_ok {
17821                    "true"
17822                } else {
17823                    "echo new-gate-failure; false"
17824                }
17825            )];
17826            runner.state.save().unwrap();
17827            runner.state = RunState::load(&runner.state.id).unwrap();
17828            runner.sync_to_base().await.unwrap();
17829            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17830            assert_ne!(head, old);
17831            // Reentry after Git updated but before verification was invalidated.
17832            runner.state = RunState::load(&runner.state.id).unwrap();
17833            runner.execute_graph().await.unwrap();
17834            assert_eq!(runner.state.gate_fixes.len(), 1);
17835            assert_eq!(runner.state.reviews.len(), 1);
17836            assert_eq!(runner.state.reviews[0].head, old);
17837            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17838            assert_eq!(runner.state.verification_history.len(), 1);
17839            let history = &runner.state.verification_history[0];
17840            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17841            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17842            assert!(history.archived_at.is_some());
17843            assert_eq!(history.fixes_spent, Some(1));
17844            assert_eq!(history.gate[0].output_tail, "old failure");
17845            let archive_events: Vec<_> = runner
17846                .state
17847                .events
17848                .iter()
17849                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17850                .collect();
17851            assert_eq!(archive_events.len(), 1);
17852            assert!(archive_events[0].message.contains(&short(&old)));
17853            assert!(archive_events[0].message.contains(&short(&head)));
17854            assert!(
17855                archive_events[0]
17856                    .message
17857                    .contains("1 gate-fix round(s) already spent stay spent")
17858            );
17859            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17860            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17861            if e2e_ok {
17862                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17863                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17864            } else {
17865                assert!(!runner.state.gate_ran);
17866            }
17867            assert_eq!(
17868                runner.state.status,
17869                if e2e_ok && gate_ok {
17870                    RunStatus::Ready
17871                } else {
17872                    RunStatus::Blocked
17873                }
17874            );
17875            if !e2e_ok || !gate_ok {
17876                let cause = runner.state.verification_blocking_cause().unwrap();
17877                assert!(cause.contains(&head));
17878                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17879                // A later successful progress event must not become the cause.
17880                runner.state.event("land", "rebased successfully");
17881                let notice = crate::notices::run_ended(&runner.state).unwrap();
17882                assert!(!format!("{notice:?}").contains("rebased successfully"));
17883                assert!(runner.state.merge.is_none());
17884            }
17885            let before = std::fs::read_to_string(&phases).unwrap();
17886            runner.state.save().unwrap();
17887            runner.state = RunState::load(&runner.state.id).unwrap();
17888            runner.review_loop().await.unwrap();
17889            runner.gate().await.unwrap();
17890            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17891            assert_eq!(runner.state.verification_history.len(), 1);
17892            let history = &runner.state.verification_history[0];
17893            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17894            assert!(history.archived_at.is_some());
17895            assert_eq!(history.fixes_spent, Some(1));
17896        }
17897    }
17898
17899    #[cfg(unix)]
17900    #[tokio::test]
17901    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17902        let (tmp, mut runner, head) = resumed_gate_fixture().await;
17903        let marker = tmp.path().join("gate-ran");
17904        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17905        runner.review_loop().await.unwrap();
17906        runner.gate().await.unwrap();
17907        assert_eq!(runner.state.status, RunStatus::Blocked);
17908        assert!(!marker.exists());
17909        assert!(runner.state.verification_history.is_empty());
17910        runner.state.gate_head = None;
17911        runner.gate().await.unwrap();
17912        assert!(marker.exists());
17913        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17914        assert_eq!(runner.state.verification_history.len(), 1);
17915        assert_eq!(runner.state.gate_fixes.len(), 1);
17916    }
17917
17918    #[cfg(unix)]
17919    #[tokio::test]
17920    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17921        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17922        runner.sync_to_base().await.unwrap();
17923        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17924        runner.state.reconcile_verification_head(&head);
17925        // Both crash-before-command and resource wait are unresolved evidence.
17926        for outcomes in [
17927            Vec::new(),
17928            vec![CommandOutcome {
17929                command: "true".into(),
17930                code: None,
17931                output_tail: "cache unavailable".into(),
17932                duration_ms: 0,
17933                resource_blocked: true,
17934            }],
17935        ] {
17936            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17937                head: head.clone(),
17938                passed: false,
17939                outcomes,
17940            });
17941            runner.state.gate_ran = false;
17942            runner.state.gate.clear();
17943            runner.state.save().unwrap();
17944            runner.state = RunState::load(&runner.state.id).unwrap();
17945            runner.merge().await.unwrap();
17946            assert!(runner.state.merge.is_none());
17947            runner.review_loop().await.unwrap();
17948            runner.gate().await.unwrap();
17949            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17950            assert!(runner.state.gate_allows_progress());
17951            assert_eq!(runner.state.gate_fixes.len(), 1);
17952            assert!(runner.state.merge.is_none());
17953        }
17954        // Moving the winner after a successful gate cannot publish an unchecked SHA.
17955        git::git(
17956            &runner.state.repo,
17957            &["commit", "--allow-empty", "-m", "external update"],
17958        )
17959        .await
17960        .unwrap();
17961        runner.merge().await.unwrap();
17962        assert!(runner.state.merge.is_none());
17963        assert_eq!(runner.state.status, RunStatus::Blocked);
17964    }
17965
17966    #[cfg(unix)]
17967    #[tokio::test]
17968    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17969        for waiting_on_gate in [false, true] {
17970            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17971            runner.sync_to_base().await.unwrap();
17972            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17973            runner.state.reconcile_verification_head(&head);
17974            if waiting_on_gate {
17975                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17976                    head: head.clone(),
17977                    passed: true,
17978                    outcomes: Vec::new(),
17979                });
17980            }
17981            let cache = tmp.path().join("cache");
17982            runner.state.config.verify.e2e =
17983                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17984            runner.state.config.verify.gate = vec!["true".into()];
17985            runner.state.config.graph.timeout_resource_wait = 1;
17986            let owner =
17987                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
17988            let guard =
17989                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
17990                    crate::cache::AcquireOutcome::Acquired(g) => g,
17991                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
17992                };
17993            runner.review_loop().await.unwrap();
17994            runner.gate().await.unwrap();
17995            assert!(!runner.state.gate_ran);
17996            assert_eq!(runner.state.gate_fixes.len(), 1);
17997            assert!(runner.state.resource_block.is_some());
17998            runner.merge().await.unwrap();
17999            assert!(runner.state.merge.is_none());
18000            drop(guard);
18001            runner.state = RunState::load(&runner.state.id).unwrap();
18002            // The driver clears the previous pass's resource stop on resume.
18003            runner.state.resource_block = None;
18004            runner.review_loop().await.unwrap();
18005            runner.gate().await.unwrap();
18006            assert!(runner.state.gate_allows_progress());
18007            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18008            assert_eq!(runner.state.gate_fixes.len(), 1);
18009        }
18010    }
18011
18012    #[cfg(unix)]
18013    #[tokio::test]
18014    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18015        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18016        runner.sync_to_base().await.unwrap();
18017        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18018        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18019        // Verification passes before pre_gate and fails on the mechanically changed head.
18020        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18021        runner.review_loop().await.unwrap();
18022        runner.gate().await.unwrap();
18023        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18024        assert_ne!(before, after);
18025        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18026        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18027        assert!(!runner.state.gate_allows_progress());
18028        runner.merge().await.unwrap();
18029        assert!(runner.state.merge.is_none());
18030        assert!(
18031            runner
18032                .state
18033                .verification_history
18034                .iter()
18035                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18036        );
18037
18038        // An incomplete review is still an independent hold on a changed head.
18039        runner.state.reviews[0].clean = false;
18040        runner.state.reviews[0].answered = 0;
18041        runner.state.config.verify.e2e = vec!["true".into()];
18042        runner.review_loop().await.unwrap();
18043        runner.gate().await.unwrap();
18044        runner.merge().await.unwrap();
18045        assert_eq!(runner.state.status, RunStatus::Blocked);
18046        assert!(runner.state.merge.is_none());
18047        assert!(
18048            runner
18049                .state
18050                .verification_blocking_cause()
18051                .unwrap()
18052                .contains("review blocked")
18053        );
18054    }
18055
18056    #[cfg(unix)]
18057    #[tokio::test]
18058    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18059        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18060        runner.state.gate_head = None;
18061        runner.state.config.verify.gate =
18062            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18063        runner.review_loop().await.unwrap();
18064        runner.gate().await.unwrap();
18065        assert_ne!(
18066            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18067            old
18068        );
18069        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18070        assert!(!runner.state.gate_allows_progress());
18071        assert_eq!(runner.state.status, RunStatus::Blocked);
18072        runner.merge().await.unwrap();
18073        assert!(runner.state.merge.is_none());
18074    }
18075
18076    #[cfg(unix)]
18077    #[tokio::test]
18078    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18079        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18080        let mut old_review = review_round(true, 0, 1, 1, false, true);
18081        old_review.head = head.clone();
18082        old_review.verified_head = Some(head.clone());
18083        runner.state.reviews = vec![old_review];
18084        runner.state.gate = vec![CommandOutcome {
18085            command: "old gate".into(),
18086            code: None,
18087            output_tail: "timed out after 2400s".into(),
18088            duration_ms: 2_400_000,
18089            resource_blocked: false,
18090        }];
18091        runner.state.gate_ran = true;
18092        runner.state.gate_head = Some(head.clone());
18093        runner.state.land_approval = Some(crate::run::LandApproval {
18094            question: "old-head-approval".into(),
18095            head: head.clone(),
18096        });
18097        runner.state.authorize_base_sync(&task).unwrap();
18098        runner.sync_to_base().await.unwrap();
18099        assert!(!runner.state.base_sync_blocked());
18100        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18101        assert_ne!(head, fresh);
18102        assert_eq!(runner.state.reviews.len(), 1);
18103        assert!(!runner.state.gate_ran);
18104        let saved = &runner.state.base_sync_recovery.authorizations[0];
18105        assert_eq!(saved.reviews[0].head, head);
18106        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18107        assert!(saved.gate_ran);
18108        assert!(!saved.gate[0].ok());
18109        runner.review_loop().await.unwrap();
18110        runner.gate().await.unwrap();
18111        assert_eq!(runner.state.reviews[0].head, head);
18112        assert_eq!(
18113            runner.state.reviews[0].verified_head.as_deref(),
18114            Some(head.as_str())
18115        );
18116        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18117        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18118        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18119        assert!(runner.state.gate_ran);
18120        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18121        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18122        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18123        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18124        assert_eq!(
18125            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18126            "retained change"
18127        );
18128    }
18129
18130    #[cfg(unix)]
18131    #[tokio::test]
18132    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18133        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18134        let queue = crate::queue::Queue::open();
18135        let mut owner = queue.get(&task).unwrap();
18136        owner.release_operator("test-resume", None);
18137        queue.put(&mut owner).unwrap();
18138        let generation = owner.resumption.as_ref().unwrap().generation;
18139        runner.state.task_instruction = Some(owner.instruction.clone());
18140        runner.state.driver_exited = true;
18141        runner.state.save().unwrap();
18142        let home = crate::run::home();
18143        let _claim = queue.claim(&task).unwrap();
18144        owner.resumption.as_mut().unwrap().at =
18145            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18146        queue.put(&mut owner).unwrap();
18147        assert!(
18148            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18149                .await
18150                .is_err(),
18151            "an approval from before the stop grants nothing"
18152        );
18153        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18154        queue.put(&mut owner).unwrap();
18155        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18156            .await
18157            .unwrap();
18158        assert!(
18159            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18160                .await
18161                .is_err()
18162        );
18163        runner.state = RunState::load(&runner.state.id).unwrap();
18164        assert_eq!(
18165            runner.state.base_sync_recovery.authorizations[0].generation,
18166            Some(generation)
18167        );
18168        runner.sync_to_base().await.unwrap();
18169        let repo = runner.state.repo.clone();
18170        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18171        git::git(&repo, &["checkout", "main"]).await.unwrap();
18172        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18173        git::git(&repo, &["add", "."]).await.unwrap();
18174        git::git(&repo, &["commit", "-m", "move base again"])
18175            .await
18176            .unwrap();
18177        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18178        git::git(&repo, &["checkout", "fix/retained"])
18179            .await
18180            .unwrap();
18181        runner.sync_to_base().await.unwrap();
18182        assert!(!runner.state.base_sync_blocked());
18183        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18184        assert_eq!(runner.state.base_episode_used(), 2);
18185        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18186        assert_ne!(fresh, head);
18187        assert!(!runner.state.base_recovery_pending());
18188        // Even replaying the generation with a later timestamp grants nothing.
18189        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18190        queue.put(&mut owner).unwrap();
18191        assert!(
18192            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18193                .await
18194                .is_err()
18195        );
18196        runner.state = RunState::load(&runner.state.id).unwrap();
18197        runner.sync_to_base().await.unwrap();
18198        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18199        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18200        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18201    }
18202
18203    #[cfg(unix)]
18204    #[tokio::test]
18205    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18206        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18207        runner.state.base_sync = None;
18208        runner.state.status = RunStatus::Blocked;
18209        let mut second = runner.state.candidates[0].clone();
18210        second.label = 'B';
18211        runner.state.candidates.push(second);
18212        assert!(runner.state.votes.is_empty());
18213        assert!(runner.state.deliberation.is_empty());
18214        runner.deliberate().await.unwrap();
18215        runner.vote().await.unwrap();
18216        assert_eq!(runner.state.status, RunStatus::Blocked);
18217        assert!(runner.state.votes.is_empty());
18218        assert!(runner.state.deliberation.is_empty());
18219    }
18220
18221    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18222    /// reads nothing else off it.
18223    fn runner_at(status: RunStatus) -> Runner {
18224        let mut state = RunState::new(
18225            PathBuf::from("/nonexistent/repo"),
18226            "main".to_owned(),
18227            "deadbeef".to_owned(),
18228            "task".to_owned(),
18229            Config::default(),
18230        );
18231        // These fixtures test execution flow, independent of host disk capacity.
18232        state.config.disk.min_free_bytes = 0;
18233        // These fixtures share one home and save concurrently. Short clock
18234        // IDs can collide, making an ownership test read another test's run.
18235        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18236        state.id = format!(
18237            "20000101-000000-{:04x}",
18238            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18239        );
18240        state.status = status;
18241        Runner {
18242            state,
18243            roles: ResolvedRoles {
18244                implementers: Vec::new(),
18245                judges: Vec::new(),
18246                reviewers: Vec::new(),
18247                fixer: None,
18248                conductor: conductor(),
18249                implementer_roster: Vec::new(),
18250                judge_roster: Vec::new(),
18251                reviewer_roster: Vec::new(),
18252            },
18253            sem: Arc::new(Semaphore::new(1)),
18254            pause: Pause::new(),
18255            interrupt: Pause::new(),
18256        }
18257    }
18258
18259    #[tokio::test]
18260    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18261        crate::run::pin_test_home();
18262        let temp = tempfile::tempdir().unwrap();
18263        let root = temp.path().canonicalize().unwrap();
18264        let repo = root.join("repo");
18265        std::fs::create_dir(&repo).unwrap();
18266        init_repo(&repo);
18267        git::git(&repo, &["config", "core.autocrlf", "false"])
18268            .await
18269            .unwrap();
18270        let mut runner = runner_at(RunStatus::Blocked);
18271        runner.state.repo = repo.clone();
18272        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18273        runner.state.config.agents = vec![conductor()];
18274        runner.state.config.graph.selected_implementer = Some("conductor".into());
18275        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18276        let (shell, command, setup_output) = if cfg!(windows) {
18277            (
18278                vec!["cmd".into(), "/C".into()],
18279                "echo ready>seat-ready",
18280                "ready\r\n",
18281            )
18282        } else {
18283            (
18284                vec!["sh".into(), "-c".into()],
18285                "printf ready > seat-ready",
18286                "ready",
18287            )
18288        };
18289        runner.state.config.verify.shell = Some(shell);
18290        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18291            run: Some(command.into()),
18292            ..Default::default()
18293        }];
18294        runner.state.save().unwrap();
18295        let path = runner.state.worktree_root().join("judge-1");
18296        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18297            .await
18298            .unwrap();
18299        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18300        git::git(&path, &["add", "."]).await.unwrap();
18301        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18302        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18303            .await
18304            .unwrap();
18305        git::git(&path, &["checkout", "--detach", &head])
18306            .await
18307            .unwrap();
18308        let evidence = crate::fold_evidence::snapshot(
18309            &runner.state,
18310            &crate::run::home().canonicalize().unwrap(),
18311            &head,
18312            64 * 1024 * 1024,
18313        )
18314        .await
18315        .unwrap();
18316        let mut contents = BTreeMap::new();
18317        contents.insert(
18318            repo.join("logs/HEAD"),
18319            format!("{head} {head} fixture\n").into_bytes(),
18320        );
18321        let record = serde_json::json!({
18322            "version":1, "run":runner.state.id, "source":path, "head":head,
18323            "evidence":evidence, "admin":{}, "admin_contents":contents,
18324            "history":{head.clone():evidence}, "phase":"evicted"
18325        });
18326        std::fs::write(
18327            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18328            serde_json::to_vec(&record).unwrap(),
18329        )
18330        .unwrap();
18331        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18332        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18333            .await
18334            .unwrap();
18335        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18336        assert!(
18337            !git::git_raw(&repo, &["cat-file", "-e", &head])
18338                .await
18339                .unwrap()
18340                .ok()
18341        );
18342        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18343        let error = resumed.execute().await.unwrap_err();
18344        assert!(
18345            error
18346                .to_string()
18347                .contains("selected implementation is held"),
18348            "{error:#}"
18349        );
18350        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18351        assert_eq!(
18352            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18353            setup_output
18354        );
18355        assert_eq!(
18356            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18357            "unrelated seat"
18358        );
18359        let error = resumed.execute().await.unwrap_err();
18360        assert!(
18361            error
18362                .to_string()
18363                .contains("selected implementation is held"),
18364            "{error:#}"
18365        );
18366        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18367        git::git(&path, &["add", "."]).await.unwrap();
18368        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18369        let next = git::git(
18370            &path,
18371            &[
18372                "commit-tree",
18373                &next_tree,
18374                "-p",
18375                &head,
18376                "-m",
18377                "next review checkpoint",
18378            ],
18379        )
18380        .await
18381        .unwrap();
18382        git::git(&path, &["checkout", "--detach", &next])
18383            .await
18384            .unwrap();
18385        resumed.state.reviews.push(
18386            serde_json::from_value(serde_json::json!({
18387                "round":1, "head":next, "reviews":[]
18388            }))
18389            .unwrap(),
18390        );
18391        resumed.state.save().unwrap();
18392        let error = resumed.execute().await.unwrap_err();
18393        assert!(
18394            error
18395                .to_string()
18396                .contains("selected implementation is held"),
18397            "{error:#}"
18398        );
18399        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18400    }
18401
18402    #[tokio::test]
18403    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18404        crate::run::pin_test_home();
18405        let mut runner = runner_at(RunStatus::Gating);
18406        runner.state.save().unwrap();
18407        let id = runner.state.id.clone();
18408        let passes = runner.state.passes.len();
18409        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18410        let mut claiming = Box::pin(runner.claim_driver());
18411        std::future::poll_fn(|cx| {
18412            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18413            std::task::Poll::Ready(())
18414        })
18415        .await;
18416        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18417        drop(owner);
18418        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18419            .await
18420            .unwrap()
18421            .unwrap();
18422        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18423        assert!(execution.runner.state.resource_block.is_none());
18424        assert_eq!(
18425            execution.runner.state.liveness(false),
18426            crate::run::Liveness::Live
18427        );
18428        drop(execution);
18429        assert!(RunState::load(&id).unwrap().driver_exited);
18430    }
18431
18432    #[tokio::test]
18433    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18434        crate::run::pin_test_home();
18435        let mut runner = runner_at(RunStatus::Gating);
18436        runner.state.save().unwrap();
18437        // run.json can commit, but the subsequent lease metadata write fails.
18438        // Cleanup must run after releasing the short ownership lock.
18439        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18440        assert!(runner.claim_driver().await.is_err());
18441        assert!(runner.state.driver_exited);
18442        assert_eq!(runner.state.driver_pid, None);
18443        let disk = RunState::load(&runner.state.id).unwrap();
18444        assert!(disk.driver_exited);
18445        assert_eq!(disk.driver_pid, None);
18446        assert_eq!(disk.status, RunStatus::Gating);
18447    }
18448
18449    #[tokio::test]
18450    async fn an_early_graph_error_releases_its_driver() {
18451        crate::run::pin_test_home();
18452        let mut runner = runner_at(RunStatus::Blocked);
18453        runner.state.config.graph.selected_implementer = Some("selected".into());
18454        runner.state.save().unwrap();
18455        assert!(runner.execute().await.is_err());
18456        let disk = RunState::load(&runner.state.id).unwrap();
18457        assert!(disk.driver_exited);
18458        assert_eq!(disk.driver_pid, None);
18459        assert_eq!(disk.status, RunStatus::Blocked);
18460    }
18461
18462    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18463        crate::run::MergeRepair {
18464            question: "repair-question".into(),
18465            head: "old-head".into(),
18466            pr: "https://github.com/o/r/pull/42".into(),
18467            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18468            gate: vec![outcome(Some(0), false)],
18469            gate_head: Some("old-head".into()),
18470            gate_fixes: Vec::new(),
18471            pre_gate: Vec::new(),
18472            phase,
18473            failure: None,
18474        }
18475    }
18476
18477    #[tokio::test]
18478    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18479        crate::run::pin_test_home();
18480        let mut runner = runner_at(RunStatus::Landing);
18481        runner.state.merge_repairs.push(merge_repair_request(
18482            crate::run::MergeRepairPhase::Requested,
18483        ));
18484        runner.state.save().unwrap();
18485        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18486        assert!(
18487            runner
18488                .execute()
18489                .await
18490                .unwrap_err()
18491                .to_string()
18492                .contains("already running")
18493        );
18494        assert_eq!(
18495            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18496            crate::run::MergeRepairPhase::Requested
18497        );
18498        drop(claim);
18499        let mut disk = runner.state.clone();
18500        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18501        disk.save().unwrap();
18502        assert!(
18503            runner
18504                .execute()
18505                .await
18506                .unwrap_err()
18507                .to_string()
18508                .contains("decision changed")
18509        );
18510        assert_eq!(
18511            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18512            crate::run::MergeRepairPhase::Complete
18513        );
18514    }
18515
18516    #[test]
18517    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18518        crate::run::pin_test_home();
18519        let mut runner = runner_at(RunStatus::Landing);
18520        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18521        runner.state.reviews = request.reviews.clone();
18522        runner.state.gate = request.gate.clone();
18523        runner.state.gate_ran = true;
18524        runner.state.land_approval = Some(crate::run::LandApproval {
18525            question: "old".into(),
18526            head: "old-head".into(),
18527        });
18528        runner.state.merge_repairs.push(request);
18529        runner.begin_merge_repair_review(0).unwrap();
18530        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18531        assert!(!runner.state.gate_status().ok());
18532        assert!(runner.state.land_approval.is_none());
18533        let disk = RunState::load(&runner.state.id).unwrap();
18534        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18535        assert!(disk.merge_repairs[0].gate[0].ok());
18536        assert_eq!(
18537            disk.merge_repairs[0].phase,
18538            crate::run::MergeRepairPhase::Reviewing
18539        );
18540        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18541        assert!(disk.reviews.is_empty());
18542    }
18543
18544    #[tokio::test]
18545    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18546        crate::run::pin_test_home();
18547        for phase in [
18548            crate::run::MergeRepairPhase::Fixing,
18549            crate::run::MergeRepairPhase::Requested,
18550        ] {
18551            let mut runner = runner_at(RunStatus::Landing);
18552            runner.state.config.graph.land_rounds = 0;
18553            runner.state.merge_repairs.push(merge_repair_request(phase));
18554            runner.state.save().unwrap();
18555            assert!(!runner.repair_merge_review().await.unwrap());
18556            assert_eq!(runner.state.status, RunStatus::Blocked);
18557            assert_eq!(
18558                runner.state.merge_repairs[0].phase,
18559                crate::run::MergeRepairPhase::Failed
18560            );
18561            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18562            assert!(
18563                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18564                    "interrupted"
18565                } else {
18566                    "budget exhausted"
18567                })
18568            );
18569            assert!(runner.state.merge.is_none());
18570        }
18571    }
18572
18573    /// `park_here` folding in the reason `Pause::park_because` recorded -
18574    /// this is what lets an operator reading a run's events tell an
18575    /// interrupt-driven park from an ordinary shutdown park.
18576    #[test]
18577    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18578        crate::run::pin_test_home();
18579        let mut runner = runner_at(RunStatus::Implementing);
18580        let interrupt = Pause::new();
18581        runner.watch_interrupt(interrupt.clone());
18582
18583        interrupt.park_because("task a1b2 asked to run first");
18584
18585        assert!(runner.park_here().expect("park_here"));
18586        assert!(runner.state.parked);
18587        let last = runner.state.events.last().expect("a park event");
18588        assert_eq!(last.node, "park");
18589        assert!(
18590            last.message.contains("task a1b2 asked to run first"),
18591            "expected the interrupt reason in {:?}",
18592            last.message
18593        );
18594    }
18595
18596    /// The cause is stored on the pass when it parks, and the first cause a
18597    /// shared pause was given is the one that stays.
18598    #[test]
18599    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18600        crate::run::pin_test_home();
18601        let mut runner = runner_at(RunStatus::Implementing);
18602        runner.state.begin_pass();
18603        let shutdown = Pause::new();
18604        runner.on_pause(shutdown.clone());
18605        let upgrade = ParkCause::Upgrade {
18606            from: "0.121.0".to_owned(),
18607            to: Some("0.121.1".to_owned()),
18608        };
18609        shutdown.park_for(upgrade.clone());
18610        shutdown.park_for(ParkCause::Operator);
18611
18612        assert!(runner.park_here().expect("park_here"));
18613        assert_eq!(
18614            runner.state.passes[0].end,
18615            Some(crate::run::PassEnd::Parked { cause: upgrade })
18616        );
18617        let last = runner.state.events.last().expect("a park event");
18618        assert!(last.message.contains("0.121.0"), "{}", last.message);
18619    }
18620
18621    #[tokio::test]
18622    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18623        crate::run::pin_test_home();
18624        let mut runner = runner_at(RunStatus::Reviewing);
18625        runner.state.save().unwrap();
18626        let mut execution = runner.claim_driver().await.expect("first execution");
18627        assert_eq!(execution.runner.state.passes.len(), 1);
18628        execution
18629            .driver
18630            .finish(&mut execution.runner.state)
18631            .await
18632            .unwrap();
18633    }
18634
18635    #[tokio::test]
18636    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18637        crate::run::pin_test_home();
18638        let mut runner = runner_at(RunStatus::Reviewing);
18639        runner.state.config.agents = vec![conductor()];
18640        runner.state.config.graph.timeout_resource_wait = 0;
18641        runner.state.save().unwrap();
18642        let id = runner.state.id.clone();
18643        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18644        assert!(runner.claim_driver().await.is_err());
18645        drop(owner);
18646        drop(runner);
18647        let mut resumed = Runner::resume(&id).unwrap();
18648        assert_eq!(resumed.state.driver_claimed, Some(false));
18649        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18650        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18651        let mut execution = resumed
18652            .claim_driver()
18653            .await
18654            .expect("retry initial claim after reload");
18655        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18656        assert_eq!(execution.runner.state.passes.len(), 1);
18657        execution
18658            .driver
18659            .finish(&mut execution.runner.state)
18660            .await
18661            .unwrap();
18662    }
18663
18664    #[tokio::test]
18665    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18666        crate::run::pin_test_home();
18667        let mut runner = runner_at(RunStatus::Reviewing);
18668        runner.state.config.agents = vec![conductor()];
18669        runner.state.save().unwrap();
18670        let id = runner.state.id.clone();
18671        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18672        let mut resumed = Runner::resume(&id).unwrap();
18673        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18674        assert!(resumed.claim_driver().await.is_err());
18675        assert!(RunState::load(&id).unwrap().passes.is_empty());
18676        drop(lease);
18677        let mut execution = resumed.claim_driver().await.unwrap();
18678        execution
18679            .driver
18680            .finish(&mut execution.runner.state)
18681            .await
18682            .unwrap();
18683    }
18684
18685    #[tokio::test]
18686    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18687        crate::run::pin_test_home();
18688        let mut runner = runner_at(RunStatus::Reviewing);
18689        let state = &mut runner.state;
18690        state.config.agents = vec![conductor()];
18691        state.driver_claimed = None;
18692        state.driver_pid = Some(std::process::id());
18693        state.driver_started_at = None;
18694        state.begin_pass();
18695        state.save().unwrap();
18696        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18697        std::fs::write(
18698            state.dir().join("driver.json"),
18699            serde_json::to_vec(&serde_json::json!({
18700                "pid": state.driver_pid, "started_at": null,
18701                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18702            }))
18703            .unwrap(),
18704        )
18705        .unwrap();
18706        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18707        drop(lease);
18708        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18709        let mut resumed = Runner::resume(&state.id).unwrap();
18710        assert!(!resumed.state.driver_exited);
18711        let mut execution = resumed
18712            .claim_driver()
18713            .await
18714            .expect("lease proves previous execution ended");
18715        assert_eq!(execution.runner.state.passes.len(), 2);
18716        execution
18717            .driver
18718            .finish(&mut execution.runner.state)
18719            .await
18720            .unwrap();
18721    }
18722
18723    #[tokio::test]
18724    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18725        crate::run::pin_test_home();
18726        let mut runner = runner_at(RunStatus::Reviewing);
18727        let mut json = serde_json::to_value(&runner.state).unwrap();
18728        json.as_object_mut().unwrap().remove("driver_claimed");
18729        runner.state = serde_json::from_value(json).unwrap();
18730        assert_eq!(runner.state.driver_claimed, None);
18731        runner.state.save().unwrap();
18732        let mut execution = runner
18733            .claim_driver()
18734            .await
18735            .expect("the execution lease, not a missing marker, excludes a second driver");
18736        assert_eq!(execution.runner.state.passes.len(), 1);
18737        execution
18738            .driver
18739            .finish(&mut execution.runner.state)
18740            .await
18741            .unwrap();
18742    }
18743
18744    #[tokio::test]
18745    async fn driver_admission_refuses_a_proven_live_driver() {
18746        crate::run::pin_test_home();
18747        let mut runner = runner_at(RunStatus::Reviewing);
18748        let pid = std::process::id();
18749        runner.state.driver_pid = Some(pid);
18750        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18751        assert!(runner.state.driver_started_at.is_some());
18752        runner.state.save().unwrap();
18753        let error = runner
18754            .claim_driver()
18755            .await
18756            .err()
18757            .expect("a live driver must refuse");
18758        assert!(error.to_string().contains("live driver"), "{error:#}");
18759        let saved = RunState::load(&runner.state.id).unwrap();
18760        assert_eq!(saved.driver_pid, Some(pid));
18761        assert_eq!(saved.status, RunStatus::Reviewing);
18762        assert!(saved.passes.is_empty());
18763    }
18764
18765    #[test]
18766    fn parking_resource_halt_persists_the_pass_once() {
18767        crate::run::pin_test_home();
18768        let mut runner = runner_at(RunStatus::Reviewing);
18769        runner.state.begin_pass();
18770        runner.state.resource_block = Some(crate::run::ResourceBlock {
18771            node: "fix".into(),
18772            kind: "shared build cache".into(),
18773            detail: "cancelled".into(),
18774            at: Timestamp::now(),
18775            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18776        });
18777        let cause = ParkCause::Upgrade {
18778            from: "old".into(),
18779            to: Some("new".into()),
18780        };
18781        runner.pause.park_for(cause.clone());
18782        // Both direct fixer exits and graph resource boundaries must complete
18783        // parking, rather than returning a stale Reviewing record.
18784        runner.stop_for_resource().unwrap();
18785        assert!(runner.resource_halt().unwrap());
18786        assert!(runner.park_here().unwrap());
18787        let saved = RunState::load(&runner.state.id).unwrap();
18788        assert!(saved.parked);
18789        assert_eq!(saved.status, RunStatus::Reviewing);
18790        assert!(saved.passes[0].ended_at.is_some());
18791        assert_eq!(
18792            saved.passes[0].end,
18793            Some(crate::run::PassEnd::Parked {
18794                cause: cause.clone()
18795            })
18796        );
18797        assert_eq!(
18798            saved.resource_block.unwrap().cause,
18799            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18800        );
18801        assert_eq!(
18802            saved
18803                .events
18804                .iter()
18805                .filter(|event| event.node == "park")
18806                .count(),
18807            1
18808        );
18809    }
18810
18811    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
18812    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
18813    /// cleared) must not make a *different* run - one only watching its own,
18814    /// unshared interrupt `Pause` - see itself as parked. If a future change
18815    /// ever collapsed these back into one handle, the interrupt scheduler
18816    /// would park every run for the rest of the daemon's life, not just the
18817    /// one it meant to interrupt.
18818    #[test]
18819    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18820        crate::run::pin_test_home();
18821        let mut runner = runner_at(RunStatus::Implementing);
18822        let shutdown = Pause::new();
18823        runner.on_pause(shutdown.clone());
18824        let interrupt = Pause::new();
18825        runner.watch_interrupt(interrupt.clone());
18826
18827        // Nobody has asked for anything yet.
18828        assert!(!runner.park_here().expect("park_here"));
18829        assert!(!runner.state.parked);
18830
18831        // Only the interrupt handle fires; the shutdown handle stays clear.
18832        interrupt.park_because("test");
18833        assert!(!shutdown.parked());
18834        assert!(runner.park_here().expect("park_here"));
18835    }
18836
18837    /// The property every prior attempt at this feature failed to pin down:
18838    /// asking a run to park while one of its nodes has a real, in-flight
18839    /// async operation running (an agent call, in production) must not cut
18840    /// that operation short. `park_here` is only ever consulted *between*
18841    /// `execute`'s node calls - see its own doc - so nothing inside a node
18842    /// can observe a park request until the node itself returns. This proves
18843    /// that structurally, with real `tokio` concurrency and a channel
18844    /// handshake (never a sleep, which would only prove "usually", not
18845    /// "cannot"): the "node" below reports that it has genuinely started,
18846    /// and only then is the park requested; the node still has to be told to
18847    /// finish before `park_here` is ever called, exactly mirroring every
18848    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
18849    /// in `execute`.
18850    #[tokio::test]
18851    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18852        crate::run::pin_test_home();
18853        let mut runner = runner_at(RunStatus::Implementing);
18854        let interrupt = Pause::new();
18855        runner.watch_interrupt(interrupt.clone());
18856
18857        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18858        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18859
18860        // Stands in for one node's in-flight agent call: it proves it has
18861        // genuinely started, then blocks - exactly as a spawned CLI process
18862        // does - until told to finish.
18863        let node = async move {
18864            started_tx.send(()).expect("send started");
18865            finish_rx.await.expect("recv finish");
18866            "node finished"
18867        };
18868
18869        let interrupter = async move {
18870            started_rx.await.expect("recv started");
18871            // The call is now genuinely in flight. Ask it to park.
18872            interrupt.park_because("higher-priority task waiting");
18873            // Nothing the node does can observe this yet - there is no
18874            // check inside it, by construction - so let the executor run
18875            // anything pending and then let the node finish on its own.
18876            tokio::task::yield_now().await;
18877            finish_tx.send(()).expect("send finish");
18878        };
18879
18880        let (node_result, ()) = tokio::join!(node, interrupter);
18881        assert_eq!(
18882            node_result, "node finished",
18883            "the in-flight call ran to completion"
18884        );
18885
18886        // Only now, at the boundary the real `execute` would check right
18887        // after this node, does the park take effect.
18888        assert!(runner.park_here().expect("park_here"));
18889        assert!(runner.state.parked);
18890    }
18891
18892    /// A run parked mid-competition carries every field it had accumulated
18893    /// through the exact same disk round-trip an ordinary resume uses -
18894    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
18895    /// Nothing about parking for an interrupt is a special case of that path;
18896    /// this is what proves it rather than assuming it.
18897    #[test]
18898    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18899        crate::run::pin_test_home();
18900        let mut runner = runner_at(RunStatus::Judging);
18901        // `Runner::resume` re-resolves roles from the saved config, which
18902        // refuses an empty roster - give it the same minimal one `conductor`
18903        // itself uses.
18904        runner.state.config.agents = vec![conductor()];
18905        runner.state.candidates = vec![Candidate {
18906            index: 0,
18907            label: 'A',
18908            agent: "alpha".to_owned(),
18909            branch: "magi/x/A".to_owned(),
18910            worktree: PathBuf::from("/nonexistent/worktree"),
18911            summary: "did the thing".to_owned(),
18912            stat: "1 file changed".to_owned(),
18913            files: 1,
18914            commits: 1,
18915            empty: false,
18916            failed: None,
18917            verified_noop: None,
18918            duration_ms: 1234,
18919            folded: false,
18920        }];
18921        let run_id = runner.state.id.clone();
18922
18923        let interrupt = Pause::new();
18924        runner.watch_interrupt(interrupt.clone());
18925        interrupt.park_because("task c3d4 asked to run first");
18926        assert!(runner.park_here().expect("park_here"));
18927
18928        let resumed = Runner::resume(&run_id).expect("resume");
18929        assert_eq!(resumed.state.candidates.len(), 1);
18930        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18931        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18932        assert_eq!(resumed.state.status, runner.state.status);
18933        assert!(
18934            resumed.state.parked,
18935            "still parked until `execute` actually walks the graph again"
18936        );
18937        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18938    }
18939
18940    /// A fresh open question on `run`, stored and handed back for assertions.
18941    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18942        let mut q = ask::Question::new(
18943            run.to_owned(),
18944            "implement".to_owned(),
18945            "impl-A".to_owned(),
18946            "Which storage backend should the cache use?".to_owned(),
18947            String::new(),
18948            vec!["SQLite".to_owned(), "Redis".to_owned()],
18949        );
18950        store.put(&mut q).unwrap();
18951        q
18952    }
18953
18954    #[test]
18955    fn a_failed_runs_open_question_is_abandoned() {
18956        ask_test_home();
18957        let store = ask::Questions::open();
18958        let mut runner = runner_at(RunStatus::Failed);
18959        let run = runner.state.id.clone();
18960        let q = ask_open_question(&store, &run);
18961
18962        runner.settle_questions();
18963
18964        let back = store.get(&q.id).unwrap();
18965        assert!(
18966            !back.status.open(),
18967            "the seat that asked died with the run; nobody is left to read an answer"
18968        );
18969        assert!(
18970            back.detail.contains(&run) && back.detail.contains("failed"),
18971            "the reason names what the run became, not just that it is gone: {}",
18972            back.detail
18973        );
18974    }
18975
18976    #[test]
18977    fn a_merged_runs_open_question_is_abandoned_too() {
18978        ask_test_home();
18979        let store = ask::Questions::open();
18980        // A run that finishes cleanly still leaves nobody to read an answer -
18981        // this is not only a failure-path cleanup.
18982        for status in [RunStatus::Merged, RunStatus::Ready] {
18983            let mut runner = runner_at(status);
18984            let run = runner.state.id.clone();
18985            let q = ask_open_question(&store, &run);
18986
18987            runner.settle_questions();
18988
18989            let back = store.get(&q.id).unwrap();
18990            assert!(
18991                !back.status.open(),
18992                "{status:?} run's question must not outlive the run"
18993            );
18994        }
18995    }
18996
18997    #[test]
18998    fn a_still_resumable_runs_open_question_is_left_alone() {
18999        ask_test_home();
19000        let store = ask::Questions::open();
19001        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19002        // review round and the seat sessions are all still on disk — so a
19003        // question asked mid-round may yet get a real answer from a real
19004        // resume. Sweeping it here would be exactly the failure mode this
19005        // whole feature exists to avoid on the other side.
19006        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19007            let mut runner = runner_at(status);
19008            let run = runner.state.id.clone();
19009            let q = ask_open_question(&store, &run);
19010
19011            runner.settle_questions();
19012
19013            let back = store.get(&q.id).unwrap();
19014            assert!(
19015                back.status.open(),
19016                "{status:?} is still alive; the question must still be waiting"
19017            );
19018        }
19019    }
19020
19021    #[test]
19022    fn settle_questions_never_touches_an_already_answered_question() {
19023        ask_test_home();
19024        let store = ask::Questions::open();
19025        let mut runner = runner_at(RunStatus::Failed);
19026        let run = runner.state.id.clone();
19027        let mut q = ask_open_question(&store, &run);
19028        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19029            .unwrap();
19030        store.put(&mut q).unwrap();
19031
19032        // Called twice, the way a crash-recovered daemon reclaim and the
19033        // graph's own cleanup both can for the same run — `abandon_for_run`
19034        // only ever touches what is still open, so this must be inert both
19035        // times, not merely the second.
19036        runner.settle_questions();
19037        runner.settle_questions();
19038
19039        let back = store.get(&q.id).unwrap();
19040        assert_eq!(
19041            back.status,
19042            ask::QuestionStatus::Answered,
19043            "a real answer is a decision on record, never overwritten by a sweep"
19044        );
19045    }
19046
19047    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19048    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19049    /// without merging, whose winner is still the operator's answer to read.
19050    /// Nothing previously called `fold_run` itself with a real `tally`, so
19051    /// this is the first test to pin down the one distinction the whole
19052    /// automatic-fold feature depends on: the winner's worktree and branch
19053    /// must survive, everything else sharing the run's worktree bay - a
19054    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19055    /// second sweep treats every non-winner directory under the bay alike -
19056    /// must not.
19057    #[tokio::test]
19058    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19059        crate::run::pin_test_home();
19060        let tmp = tempfile::tempdir().expect("tempdir");
19061        let repo = tmp.path().join("repo");
19062        std::fs::create_dir_all(&repo).unwrap();
19063        init_repo(&repo);
19064
19065        let mut config = Config::default();
19066        config.graph.worktree_root = Some(tmp.path().join("wt"));
19067
19068        let mut state = RunState::new(
19069            repo.clone(),
19070            "main".to_owned(),
19071            "deadbeef".to_owned(),
19072            "task".to_owned(),
19073            config,
19074        );
19075        let root = state.worktree_root();
19076        let wt_a = root.join("cand-A");
19077        let wt_b = root.join("cand-B");
19078        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19079            .await
19080            .expect("worktree A");
19081        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19082            .await
19083            .expect("worktree B");
19084
19085        state.candidates = vec![
19086            Candidate {
19087                index: 0,
19088                label: 'A',
19089                agent: "alpha".to_owned(),
19090                branch: "magi/x/A".to_owned(),
19091                worktree: wt_a.clone(),
19092                summary: String::new(),
19093                stat: String::new(),
19094                files: 0,
19095                commits: 0,
19096                empty: false,
19097                failed: None,
19098                verified_noop: None,
19099                duration_ms: 0,
19100                folded: false,
19101            },
19102            Candidate {
19103                index: 1,
19104                label: 'B',
19105                agent: "beta".to_owned(),
19106                branch: "magi/x/B".to_owned(),
19107                worktree: wt_b.clone(),
19108                summary: String::new(),
19109                stat: String::new(),
19110                files: 0,
19111                commits: 0,
19112                empty: false,
19113                failed: None,
19114                verified_noop: None,
19115                duration_ms: 0,
19116                folded: false,
19117            },
19118        ];
19119        state.tally = Some(Tally {
19120            first_choice: BTreeMap::from([('A', 1)]),
19121            borda: BTreeMap::new(),
19122            winner: 'A',
19123            rankings: 1,
19124            unanimous_initial: true,
19125            deliberated: false,
19126            changed_votes: 0,
19127            unanimous_final: true,
19128            tie_break: None,
19129            judges: 1,
19130            present: 1,
19131            quorum: 1,
19132            met_quorum: true,
19133            uncontested: None,
19134        });
19135        state.status = RunStatus::Ready;
19136
19137        fold_run(&mut state, false, &crate::run::home())
19138            .await
19139            .expect("fold_run");
19140
19141        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19142        assert!(
19143            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19144            "the unmerged winner's branch survives"
19145        );
19146        assert!(
19147            !state.candidates[0].folded,
19148            "the winner is not marked folded"
19149        );
19150
19151        assert!(!wt_b.exists(), "the loser's worktree is removed");
19152        assert!(
19153            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19154            "the loser's branch is removed"
19155        );
19156        assert!(state.candidates[1].folded, "the loser is marked folded");
19157    }
19158
19159    #[tokio::test]
19160    #[cfg(unix)]
19161    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19162        use std::os::fd::AsRawFd as _;
19163        use std::time::Duration;
19164
19165        let tmp = tempfile::tempdir().unwrap();
19166        let repo = tmp.path().join("repo");
19167        std::fs::create_dir_all(&repo).unwrap();
19168        init_repo(&repo);
19169        let common = repo.join(".git");
19170        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19171        let home = tmp.path().join("home");
19172        let mut config = Config::default();
19173        config.coordination = crate::config::test_coordination();
19174        config.graph.worktree_root = Some(tmp.path().join("wt"));
19175        config.graph.timeout_resource_wait = 1;
19176        let mut state = RunState::new(
19177            repo.clone(),
19178            "main".into(),
19179            "deadbeef".into(),
19180            "task".into(),
19181            config,
19182        );
19183        let root = state.worktree_root();
19184        let candidate = root.join("cand-A");
19185        let reviewer = root.join("review-1");
19186        let branch = state.branch_for('A');
19187        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19188            .await
19189            .unwrap();
19190        git::worktree_add_detached(&repo, &reviewer, "main")
19191            .await
19192            .unwrap();
19193        git::acquire_worktree_config(&repo).await.unwrap();
19194        state.enabled_worktree_config = true;
19195        state.status = RunStatus::Ready;
19196        state.candidates.push(Candidate {
19197            index: 0,
19198            label: 'A',
19199            agent: "implement".into(),
19200            branch: branch.clone(),
19201            worktree: candidate.clone(),
19202            summary: String::new(),
19203            stat: String::new(),
19204            files: 0,
19205            commits: 0,
19206            empty: false,
19207            failed: None,
19208            verified_noop: None,
19209            duration_ms: 0,
19210            folded: false,
19211        });
19212        state.save_under(&home).unwrap();
19213        let config_before = std::fs::read(common.join("config")).unwrap();
19214        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19215            .await
19216            .unwrap();
19217        // A separate OS handle bypasses the launcher's local gate, exactly as
19218        // an external updater does, and must still prevent all folding work.
19219        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19220        assert_eq!(
19221            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19222            0
19223        );
19224        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19225        assert!(
19226            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19227                .await
19228                .is_err()
19229        );
19230        assert!(candidate.exists());
19231        assert!(reviewer.exists());
19232        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19233        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19234        assert_eq!(
19235            git::git(&repo, &["worktree", "list", "--porcelain"])
19236                .await
19237                .unwrap(),
19238            worktrees_before
19239        );
19240        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19241            .await
19242            .unwrap()
19243            .unwrap_err();
19244        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19245        drop(fold);
19246        assert!(candidate.exists() && reviewer.exists());
19247        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19248        assert_eq!(
19249            state.resource_block.as_ref().unwrap().kind,
19250            "repository coordination"
19251        );
19252        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19253        drop(external);
19254        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19255        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19256            .await
19257            .unwrap()
19258            .unwrap();
19259        drop(fold);
19260        assert!(removed.contains(&branch));
19261        assert!(!root.exists());
19262        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19263        assert!(state.candidates[0].folded);
19264        assert!(!state.enabled_worktree_config);
19265        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19266        let enabled = git::git_raw(
19267            &repo,
19268            &["config", "--local", "--get", "extensions.worktreeConfig"],
19269        )
19270        .await
19271        .unwrap();
19272        assert!(!enabled.ok(), "the last config reference was released");
19273    }
19274
19275    /// A branch handed to a later run is that run's (and its pull request's):
19276    /// folding the run that released it must not delete it.
19277    #[tokio::test]
19278    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19279        let tmp = tempfile::tempdir().expect("tempdir");
19280        let repo = tmp.path().join("repo");
19281        std::fs::create_dir_all(&repo).unwrap();
19282        init_repo(&repo);
19283        let home = tmp.path().join("home");
19284
19285        let mut config = Config::default();
19286        config.graph.worktree_root = Some(tmp.path().join("wt"));
19287        let mut state = RunState::new(
19288            repo.clone(),
19289            "main".to_owned(),
19290            "deadbeef".to_owned(),
19291            "task".to_owned(),
19292            config,
19293        );
19294        // The worktree is already gone (released); the branch survives.
19295        git::git(&repo, &["branch", "magi/x/A", "main"])
19296            .await
19297            .expect("branch");
19298        state.candidates = vec![Candidate {
19299            index: 0,
19300            label: 'A',
19301            agent: "alpha".to_owned(),
19302            branch: "magi/x/A".to_owned(),
19303            worktree: state.worktree_root().join("cand-A"),
19304            summary: String::new(),
19305            stat: String::new(),
19306            files: 0,
19307            commits: 0,
19308            empty: false,
19309            failed: None,
19310            verified_noop: None,
19311            duration_ms: 0,
19312            folded: true,
19313        }];
19314        state.released_to = Some("20260901-000000-new1".to_owned());
19315        state.released_branches = vec!["magi/x/A".to_owned()];
19316
19317        fold_run(&mut state, true, &home).await.expect("fold_run");
19318        state.status = RunStatus::Merged;
19319        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19320        state.merge = Some(crate::run::MergeOutcome {
19321            mode: MergeMode::Pr,
19322            ok: true,
19323            detail: "confirmed merged".into(),
19324            empty: false,
19325        });
19326        fold_run(&mut state, true, &home)
19327            .await
19328            .expect("merged fold");
19329
19330        assert!(
19331            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19332            "the handed-over branch survives a fold"
19333        );
19334    }
19335
19336    /// A winner with nothing ahead of the base is caught before `gh` is ever
19337    /// asked for a pull request, and the message carries what the task's
19338    /// references resolved to.
19339    #[tokio::test]
19340    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19341        let tmp = tempfile::tempdir().expect("tempdir");
19342        let repo = tmp.path().join("repo");
19343        std::fs::create_dir_all(&repo).unwrap();
19344        init_repo(&repo);
19345        let run = |args: &[&str]| {
19346            let out = std::process::Command::new("git")
19347                .quiet()
19348                .args(args)
19349                .current_dir(&repo)
19350                .output()
19351                .expect("spawn git");
19352            assert!(out.status.success(), "git {args:?}");
19353        };
19354        run(&["branch", "magi/x/A"]);
19355        run(&["checkout", "-q", "-b", "magi/x/B"]);
19356        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19357        run(&["add", "-A"]);
19358        run(&["commit", "-q", "-m", "work"]);
19359        run(&["checkout", "-q", "main"]);
19360
19361        // A pull request is compared against the remote's base, so the
19362        // fixture needs one. Before it exists nothing can be read, and the
19363        // answer must be "not empty".
19364        let probe = RunState::new(
19365            repo.clone(),
19366            "main".to_owned(),
19367            "deadbeef".to_owned(),
19368            "task".to_owned(),
19369            Config::default(),
19370        );
19371        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19372        let bare = tmp.path().join("origin.git");
19373        let out = std::process::Command::new("git")
19374            .quiet()
19375            .args(["init", "-q", "--bare"])
19376            .arg(&bare)
19377            .output()
19378            .expect("spawn git");
19379        assert!(out.status.success(), "git init --bare");
19380        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19381        run(&["push", "-q", "origin", "main"]);
19382
19383        let mut state = RunState::new(
19384            repo.clone(),
19385            "main".to_owned(),
19386            "deadbeef".to_owned(),
19387            "task".to_owned(),
19388            Config::default(),
19389        );
19390        state.seeds = vec![refs::Seed {
19391            token: "magi/27b2/A".to_owned(),
19392            kind: refs::SeedKind::Unresolved,
19393            sha: String::new(),
19394            branch: true,
19395            detail: "no branch or commit named magi/27b2/A".to_owned(),
19396        }];
19397
19398        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19399        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19400        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19401        let detail = empty_candidate_detail(&state, "main");
19402        assert!(detail.starts_with("empty candidate"), "{detail}");
19403        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19404        assert!(detail.contains("--start-from"), "{detail}");
19405    }
19406
19407    /// `status == Ready` used to be read as "this is the harmless
19408    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19409    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19410    /// run's PR was closed without merging — and reentering `merge` with
19411    /// `mode` still `Pr` does not know the difference, so it pushed and
19412    /// opened a second pull request. `mode == Local` reproduces the same
19413    /// blind spot without a network call: reentry must not attempt another
19414    /// git merge once this node has already recorded an outcome.
19415    #[tokio::test]
19416    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19417        let tmp = tempfile::tempdir().expect("tempdir");
19418        let repo = tmp.path().join("repo");
19419        std::fs::create_dir_all(&repo).unwrap();
19420        init_repo(&repo);
19421
19422        let mut config = Config::default();
19423        config.merge.mode = MergeMode::Local;
19424
19425        let mut state = RunState::new(
19426            repo.clone(),
19427            "main".to_owned(),
19428            "deadbeef".to_owned(),
19429            "task".to_owned(),
19430            config,
19431        );
19432        state.candidates = vec![Candidate {
19433            index: 0,
19434            label: 'A',
19435            agent: "alpha".to_owned(),
19436            branch: "does-not-exist".to_owned(),
19437            worktree: repo.clone(),
19438            summary: String::new(),
19439            stat: String::new(),
19440            files: 0,
19441            commits: 0,
19442            empty: false,
19443            failed: None,
19444            verified_noop: None,
19445            duration_ms: 0,
19446            folded: false,
19447        }];
19448        state.tally = Some(Tally {
19449            first_choice: BTreeMap::from([('A', 1)]),
19450            borda: BTreeMap::new(),
19451            winner: 'A',
19452            rankings: 1,
19453            unanimous_initial: true,
19454            deliberated: false,
19455            changed_votes: 0,
19456            unanimous_final: true,
19457            tie_break: None,
19458            judges: 0,
19459            present: 0,
19460            quorum: 0,
19461            met_quorum: true,
19462            uncontested: Some("only candidate A produced a change".to_owned()),
19463        });
19464        state.reviews = vec![ReviewRound {
19465            round: 1,
19466            head: "deadbeef".to_owned(),
19467            verified_head: None,
19468            verified_at: None,
19469            reviews: Vec::new(),
19470            e2e: Vec::new(),
19471            fix: None,
19472            blocking: 0,
19473            answered: 0,
19474            expected: 0,
19475            clean: true,
19476            verify_retried: false,
19477            e2e_deferred: false,
19478            e2e_defer_reason: None,
19479            progressed: false,
19480            vote_split: false,
19481            reconsideration: Vec::new(),
19482            verdict: None,
19483        }];
19484        state.gate = vec![CommandOutcome {
19485            command: "test".to_owned(),
19486            code: Some(0),
19487            output_tail: String::new(),
19488            duration_ms: 0,
19489            resource_blocked: false,
19490        }];
19491        state.gate_ran = true;
19492        // Reached its conclusion already — e.g. `land` closing the PR without
19493        // merging it, which (like the honest `MergeMode::None` path) leaves
19494        // `status` at `Ready`. The recorded outcome is what actually marks
19495        // this node done.
19496        state.status = RunStatus::Ready;
19497        state.merge = Some(MergeOutcome {
19498            mode: MergeMode::Local,
19499            ok: false,
19500            detail: "already concluded".to_owned(),
19501            empty: false,
19502        });
19503
19504        let mut runner = Runner {
19505            state,
19506            roles: ResolvedRoles {
19507                implementers: Vec::new(),
19508                judges: Vec::new(),
19509                reviewers: Vec::new(),
19510                fixer: None,
19511                conductor: conductor(),
19512                implementer_roster: Vec::new(),
19513                judge_roster: Vec::new(),
19514                reviewer_roster: Vec::new(),
19515            },
19516            sem: Arc::new(Semaphore::new(1)),
19517            pause: Pause::new(),
19518            interrupt: Pause::new(),
19519        };
19520
19521        runner.merge().await.expect("merge");
19522
19523        assert_eq!(
19524            runner.state.status,
19525            RunStatus::Ready,
19526            "a concluded run's status must not change on reentry"
19527        );
19528        assert_eq!(
19529            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19530            Some("already concluded"),
19531            "merge must not run again once the node already recorded an outcome"
19532        );
19533    }
19534
19535    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19536    /// when its last attempt was resource-blocked (the shared build cache
19537    /// could not be acquired or confirmed fresh in time - see
19538    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19539    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19540    /// a run merge a tree the gate never actually checked - exactly the case
19541    /// a contended cache produces on every retry until it clears. `merge`
19542    /// must refuse until `gate` has actually recorded an attempt.
19543    #[tokio::test]
19544    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19545        let tmp = tempfile::tempdir().expect("tempdir");
19546        let repo = tmp.path().join("repo");
19547        std::fs::create_dir_all(&repo).unwrap();
19548        init_repo(&repo);
19549
19550        let mut config = Config::default();
19551        config.merge.mode = MergeMode::Local;
19552
19553        let mut state = RunState::new(
19554            repo.clone(),
19555            "main".to_owned(),
19556            "deadbeef".to_owned(),
19557            "task".to_owned(),
19558            config,
19559        );
19560        state.candidates = vec![Candidate {
19561            index: 0,
19562            label: 'A',
19563            agent: "alpha".to_owned(),
19564            branch: "does-not-exist".to_owned(),
19565            worktree: repo.clone(),
19566            summary: String::new(),
19567            stat: String::new(),
19568            files: 0,
19569            commits: 0,
19570            empty: false,
19571            failed: None,
19572            verified_noop: None,
19573            duration_ms: 0,
19574            folded: false,
19575        }];
19576        state.tally = Some(Tally {
19577            first_choice: BTreeMap::from([('A', 1)]),
19578            borda: BTreeMap::new(),
19579            winner: 'A',
19580            rankings: 1,
19581            unanimous_initial: true,
19582            deliberated: false,
19583            changed_votes: 0,
19584            unanimous_final: true,
19585            tie_break: None,
19586            judges: 0,
19587            present: 0,
19588            quorum: 0,
19589            met_quorum: true,
19590            uncontested: Some("only candidate A produced a change".to_owned()),
19591        });
19592        state.reviews = vec![ReviewRound {
19593            round: 1,
19594            head: "deadbeef".to_owned(),
19595            verified_head: None,
19596            verified_at: None,
19597            reviews: Vec::new(),
19598            e2e: Vec::new(),
19599            fix: None,
19600            blocking: 0,
19601            answered: 0,
19602            expected: 0,
19603            clean: true,
19604            verify_retried: false,
19605            e2e_deferred: false,
19606            e2e_defer_reason: None,
19607            progressed: false,
19608            vote_split: false,
19609            reconsideration: Vec::new(),
19610            verdict: None,
19611        }];
19612        // The point: `gate` has not recorded anything yet.
19613        state.gate = Vec::new();
19614        state.gate_ran = false;
19615        state.status = RunStatus::Gating;
19616
19617        let mut runner = Runner {
19618            state,
19619            roles: ResolvedRoles {
19620                implementers: Vec::new(),
19621                judges: Vec::new(),
19622                reviewers: Vec::new(),
19623                fixer: None,
19624                conductor: conductor(),
19625                implementer_roster: Vec::new(),
19626                judge_roster: Vec::new(),
19627                reviewer_roster: Vec::new(),
19628            },
19629            sem: Arc::new(Semaphore::new(1)),
19630            pause: Pause::new(),
19631            interrupt: Pause::new(),
19632        };
19633
19634        runner.merge().await.expect("merge");
19635
19636        assert!(
19637            runner.state.merge.is_none(),
19638            "an empty gate must never be read as a passing one: {:?}",
19639            runner.state.merge
19640        );
19641    }
19642
19643    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19644    /// commands configured and `merge.mode` is `none` (a review-only run).
19645    /// `gate` must still record a real attempt — zero commands, vacuously
19646    /// passed — rather than leaving `state.gate` empty in a way `merge`
19647    /// cannot tell apart from "never ran"; otherwise the run reaches
19648    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19649    #[tokio::test]
19650    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19651        let tmp = tempfile::tempdir().expect("tempdir");
19652        let repo = tmp.path().join("repo");
19653        std::fs::create_dir_all(&repo).unwrap();
19654        init_repo(&repo);
19655
19656        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19657        let config = Config::default();
19658
19659        let mut state = RunState::new(
19660            repo.clone(),
19661            "main".to_owned(),
19662            "deadbeef".to_owned(),
19663            "task".to_owned(),
19664            config,
19665        );
19666        state.candidates = vec![Candidate {
19667            index: 0,
19668            label: 'A',
19669            agent: "alpha".to_owned(),
19670            branch: "does-not-exist".to_owned(),
19671            worktree: repo.clone(),
19672            summary: String::new(),
19673            stat: String::new(),
19674            files: 0,
19675            commits: 0,
19676            empty: false,
19677            failed: None,
19678            verified_noop: None,
19679            duration_ms: 0,
19680            folded: false,
19681        }];
19682        state.tally = Some(Tally {
19683            first_choice: BTreeMap::from([('A', 1)]),
19684            borda: BTreeMap::new(),
19685            winner: 'A',
19686            rankings: 1,
19687            unanimous_initial: true,
19688            deliberated: false,
19689            changed_votes: 0,
19690            unanimous_final: true,
19691            tie_break: None,
19692            judges: 0,
19693            present: 0,
19694            quorum: 0,
19695            met_quorum: true,
19696            uncontested: Some("only candidate A produced a change".to_owned()),
19697        });
19698        state.reviews = vec![ReviewRound {
19699            round: 1,
19700            head: "deadbeef".to_owned(),
19701            verified_head: None,
19702            verified_at: None,
19703            reviews: Vec::new(),
19704            e2e: Vec::new(),
19705            fix: None,
19706            blocking: 0,
19707            answered: 0,
19708            expected: 0,
19709            clean: true,
19710            verify_retried: false,
19711            e2e_deferred: false,
19712            e2e_defer_reason: None,
19713            progressed: false,
19714            vote_split: false,
19715            reconsideration: Vec::new(),
19716            verdict: None,
19717        }];
19718
19719        let mut runner = Runner {
19720            state,
19721            roles: ResolvedRoles {
19722                implementers: Vec::new(),
19723                judges: Vec::new(),
19724                reviewers: Vec::new(),
19725                fixer: None,
19726                conductor: conductor(),
19727                implementer_roster: Vec::new(),
19728                judge_roster: Vec::new(),
19729                reviewer_roster: Vec::new(),
19730            },
19731            sem: Arc::new(Semaphore::new(1)),
19732            pause: Pause::new(),
19733            interrupt: Pause::new(),
19734        };
19735
19736        runner.gate().await.expect("gate");
19737        assert!(
19738            runner.state.gate_ran,
19739            "zero configured commands is still a real attempt, not an unrun gate"
19740        );
19741        assert!(runner.state.gate.is_empty());
19742        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19743        assert_ne!(
19744            runner.state.status,
19745            RunStatus::Blocked,
19746            "a gate with nothing to check must not read as failed"
19747        );
19748
19749        runner.merge().await.expect("merge");
19750        assert_eq!(
19751            runner.state.status,
19752            RunStatus::Ready,
19753            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19754        );
19755    }
19756
19757    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19758    /// `verify.gate` (so the e2e leg and the final gate never build against
19759    /// different directories). With zero `verify.gate` commands but a
19760    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19761    /// that lease before discovering it had nothing to run - so a repo with
19762    /// no gate commands could come back `resource_blocked` (and therefore
19763    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19764    /// cache, exactly the contention this run's own zero commands could
19765    /// never have touched. `gate` must recognise there is nothing to check
19766    /// before it ever asks for the lease.
19767    #[tokio::test]
19768    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19769        crate::run::pin_test_home();
19770        let home = crate::run::home();
19771
19772        let tmp = tempfile::tempdir().expect("tempdir");
19773        let repo = tmp.path().join("repo");
19774        std::fs::create_dir_all(&repo).unwrap();
19775        init_repo(&repo);
19776        // Unique to this test, so holding its lease cannot collide with
19777        // another test sharing the same process-wide `home`.
19778        let cache_dir = tmp.path().join("target");
19779
19780        let mut config = Config::default();
19781        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19782        // `verify.gate` stays empty (the default). Any accidental cache
19783        // acquisition must report contention immediately, regardless of how
19784        // long unrelated Git subprocesses take under parallel test load.
19785        config.graph.timeout_verify = Some(2);
19786        config.graph.timeout_resource_wait = 0;
19787
19788        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19789        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19790            .expect("no io error acquiring directly")
19791        {
19792            crate::cache::AcquireOutcome::Acquired(g) => g,
19793            crate::cache::AcquireOutcome::Busy(b) => {
19794                panic!("expected the direct acquire to win the lease first: {b:?}")
19795            }
19796        };
19797
19798        let mut state = RunState::new(
19799            repo.clone(),
19800            "main".to_owned(),
19801            "deadbeef".to_owned(),
19802            "task".to_owned(),
19803            config,
19804        );
19805        state.candidates = vec![Candidate {
19806            index: 0,
19807            label: 'A',
19808            agent: "alpha".to_owned(),
19809            branch: "does-not-exist".to_owned(),
19810            worktree: repo.clone(),
19811            summary: String::new(),
19812            stat: String::new(),
19813            files: 0,
19814            commits: 0,
19815            empty: false,
19816            failed: None,
19817            verified_noop: None,
19818            duration_ms: 0,
19819            folded: false,
19820        }];
19821        state.tally = Some(Tally {
19822            first_choice: BTreeMap::from([('A', 1)]),
19823            borda: BTreeMap::new(),
19824            winner: 'A',
19825            rankings: 1,
19826            unanimous_initial: true,
19827            deliberated: false,
19828            changed_votes: 0,
19829            unanimous_final: true,
19830            tie_break: None,
19831            judges: 0,
19832            present: 0,
19833            quorum: 0,
19834            met_quorum: true,
19835            uncontested: Some("only candidate A produced a change".to_owned()),
19836        });
19837        state.reviews = vec![ReviewRound {
19838            round: 1,
19839            head: "deadbeef".to_owned(),
19840            verified_head: None,
19841            verified_at: None,
19842            reviews: Vec::new(),
19843            e2e: Vec::new(),
19844            fix: None,
19845            blocking: 0,
19846            answered: 0,
19847            expected: 0,
19848            clean: true,
19849            verify_retried: false,
19850            e2e_deferred: false,
19851            e2e_defer_reason: None,
19852            progressed: false,
19853            vote_split: false,
19854            reconsideration: Vec::new(),
19855            verdict: None,
19856        }];
19857
19858        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19859        state.reviews[0].e2e = vec![CommandOutcome {
19860            command: state.config.verify.e2e[0].clone(),
19861            code: Some(0),
19862            output_tail: String::new(),
19863            duration_ms: 0,
19864            resource_blocked: false,
19865        }];
19866        let mut runner = Runner {
19867            state,
19868            roles: ResolvedRoles {
19869                implementers: Vec::new(),
19870                judges: Vec::new(),
19871                reviewers: Vec::new(),
19872                fixer: None,
19873                conductor: conductor(),
19874                implementer_roster: Vec::new(),
19875                judge_roster: Vec::new(),
19876                reviewer_roster: Vec::new(),
19877            },
19878            sem: Arc::new(Semaphore::new(1)),
19879            pause: Pause::new(),
19880            interrupt: Pause::new(),
19881        };
19882
19883        runner.gate().await.expect("gate");
19884        assert!(runner.state.resource_block.is_none());
19885        assert!(
19886            !runner
19887                .state
19888                .events
19889                .iter()
19890                .any(|event| { event.message.contains("waiting for the shared build cache") }),
19891            "a gate with nothing to run must never request the held cache lease"
19892        );
19893        assert!(
19894            runner.state.gate_ran,
19895            "zero commands is still a real, immediate attempt"
19896        );
19897        assert!(runner.state.gate.is_empty());
19898        assert_ne!(
19899            runner.state.status,
19900            RunStatus::Blocked,
19901            "must not read as resource-blocked on a lease it never asked for"
19902        );
19903    }
19904
19905    /// A separate process carries a live worker's environment throughout all
19906    /// verification entry points; the parallel harness never changes its env.
19907    #[cfg(unix)]
19908    #[tokio::test]
19909    async fn verification_is_standalone_inside_a_live_worker() {
19910        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19911        if std::env::var_os(CHILD).is_none() {
19912            let home = tempfile::tempdir().unwrap();
19913            let generation = crate::worker::pin_current(home.path()).unwrap();
19914            let output = std::process::Command::new(std::env::current_exe().unwrap())
19915                .args([
19916                    "--exact",
19917                    "graph::tests::verification_is_standalone_inside_a_live_worker",
19918                    "--nocapture",
19919                ])
19920                .env(CHILD, "1")
19921                .env("MAGI_HOME", home.path())
19922                .env("MAGI_WORKER_HOME", home.path())
19923                .env("MAGI_WORKER_ID", "abcdef")
19924                .env("MAGI_WORKER_GENERATION", &generation.id)
19925                .env("MAGI_WORKER_EXE", &generation.executable)
19926                .env("MAGI_RUN", "verification-parent")
19927                .env("MAGI_NODE", "gate")
19928                .env("MAGI_SEAT", "worker-fixture")
19929                .env("MAGI_PROCESS_OWNER", "fixture-owner")
19930                .env("MAGI_NO_AUTOUPDATE", "0")
19931                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19932                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19933                .env(
19934                    "VERIFY_EXPECT_TOOLCHAIN",
19935                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19936                )
19937                .env(
19938                    "VERIFY_EXPECT_CACHE",
19939                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19940                )
19941                .output()
19942                .unwrap();
19943            assert!(
19944                output.status.success()
19945                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19946                "{}\n{}",
19947                String::from_utf8_lossy(&output.stdout),
19948                String::from_utf8_lossy(&output.stderr)
19949            );
19950            return;
19951        }
19952        let home = crate::run::home();
19953        let directory = home.join("workers/jobs/abcdef");
19954        std::fs::create_dir_all(&directory).unwrap();
19955        let job = crate::worker::Job {
19956            id: "abcdef".into(),
19957            sequence: 1,
19958            kind: "task".into(),
19959            key: "task:verification".into(),
19960            payload: serde_json::json!({}),
19961            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19962            version: Some(env!("CARGO_PKG_VERSION").into()),
19963            status: crate::worker::Status::Running,
19964            result: None,
19965            pid: Some(std::process::id()),
19966            created_at: 0,
19967            finished_at: None,
19968        };
19969        std::fs::write(
19970            directory.join("job.json"),
19971            serde_json::to_vec(&job).unwrap(),
19972        )
19973        .unwrap();
19974        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19975        let assert_worker = || {
19976            assert_eq!(
19977                crate::worker::current_job(&home).unwrap().unwrap().id,
19978                job.id
19979            );
19980            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19981            assert_eq!(
19982                crate::worker::helper_executable(&home).unwrap(),
19983                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19984            );
19985            assert!(
19986                crate::updater::Checker::new(&crate::config::Update {
19987                    mode: crate::config::UpdateMode::Notify,
19988                    interval: None,
19989                    auto_restart: false,
19990                })
19991                .is_none(),
19992                "the driver remains pinned, even while tests are standalone"
19993            );
19994        };
19995        assert_worker();
19996        let repo = home.join("repo");
19997        std::fs::create_dir_all(&repo).unwrap();
19998        init_repo(&repo);
19999        // Keep storage, seat/control context, containment and tool lookup.
20000        // No inherited values (or credentials) are printed on success.
20001        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20002            test -z "${MAGI_WORKER_GENERATION+x}" &&
20003            test -z "${MAGI_WORKER_HOME+x}" &&
20004            test -z "${MAGI_WORKER_EXE+x}" &&
20005            test "$MAGI_HOME/repo" -ef . &&
20006            test "$MAGI_RUN" = verification-parent &&
20007            test "$MAGI_NODE" = gate &&
20008            test "$MAGI_SEAT" = worker-fixture &&
20009            test -n "$MAGI_PROCESS_OWNER" &&
20010            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20011            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20012            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20013            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20014        // Negative control: an ordinary child inherits the contamination.
20015        assert!(
20016            !std::process::Command::new("sh")
20017                .args(["-c", probe])
20018                .current_dir(&repo)
20019                .status()
20020                .unwrap()
20021                .success()
20022        );
20023        let unrelated_home = tempfile::tempdir().unwrap();
20024        assert!(
20025            crate::worker::current_job(unrelated_home.path()).is_err(),
20026            "an inherited worker id cannot be resolved in a unit fixture's home"
20027        );
20028        let mut runner = runner_at(RunStatus::Gating);
20029        runner.state.repo = repo.clone();
20030        runner.state.config.disk.min_free_bytes = 0;
20031        runner.state.config.coordination = crate::config::test_coordination();
20032        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20033        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20034        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20035        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20036        let winner = Candidate {
20037            index: 0,
20038            label: 'A',
20039            agent: "fixture".into(),
20040            branch: "main".into(),
20041            worktree: repo.clone(),
20042            summary: String::new(),
20043            stat: String::new(),
20044            files: 0,
20045            commits: 0,
20046            empty: false,
20047            failed: None,
20048            verified_noop: None,
20049            duration_ms: 0,
20050            folded: false,
20051        };
20052        let (e2e, retried, pids) = run_e2e_with_retry(
20053            &mut runner.state,
20054            &["sh".into(), "-c".into()],
20055            &[format!("{probe} && printf initial")],
20056            &repo,
20057            Duration::from_secs(10),
20058            "initial",
20059        )
20060        .await;
20061        assert!(!retried && pids.is_empty());
20062        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20063        assert_eq!(e2e[0].output_tail, "initial");
20064        let gate = runner.run_gate(&winner).await.unwrap();
20065        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20066        // An interrupted repair must recheck e2e from saved state. Landing
20067        // reuses that state and must run both pre-gate and gate again.
20068        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20069            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20070            passed: false,
20071            outcomes: Vec::new(),
20072        });
20073        runner.state.save().unwrap();
20074        let mut resumed = RunState::load(&runner.state.id).unwrap();
20075        let wait = crate::repo_lock::WaitPolicy::default();
20076        assert!(matches!(
20077            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20078                .await
20079                .unwrap(),
20080            GateFix::Retry
20081        ));
20082        let saved = RunState::load(&resumed.id).unwrap();
20083        let repair = saved.gate_repair_e2e.unwrap();
20084        assert!(repair.passed);
20085        assert_eq!(repair.outcomes.len(), 1);
20086        assert!(repair.outcomes[0].ok());
20087        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20088            .await
20089            .unwrap();
20090        assert_eq!(landing.len(), 1);
20091        assert!(landing[0].ok(), "{landing:?}");
20092        assert_eq!(
20093            std::fs::read_to_string(repo.join("phases")).unwrap(),
20094            "pre\ngate\ne2e\npre\ngate\n"
20095        );
20096        assert_worker();
20097    }
20098
20099    #[cfg(unix)]
20100    #[tokio::test]
20101    async fn verification_keeps_the_panic_and_the_full_output() {
20102        crate::run::pin_test_home();
20103        let tmp = tempfile::tempdir().unwrap();
20104        let mut config = Config::default();
20105        config.disk.min_free_bytes = 0;
20106        let mut state = RunState::new(
20107            tmp.path().to_owned(),
20108            "main".into(),
20109            "base".into(),
20110            "verification evidence".into(),
20111            config,
20112        );
20113        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20114                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20115                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20116        let mut names = Vec::new();
20117        for _ in 0..2 {
20118            let (outcomes, _) = run_commands(
20119                &mut state,
20120                "gate",
20121                "gate",
20122                0,
20123                &["sh".into(), "-c".into()],
20124                &[script.into()],
20125                tmp.path(),
20126                Duration::from_secs(60),
20127            )
20128            .await;
20129            let tail = &outcomes[0].output_tail;
20130            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20131            assert!(tail.contains("boom assertion"));
20132            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20133            let line = tail.lines().next().unwrap();
20134            names.push(line.to_owned());
20135        }
20136        assert_ne!(names[0], names[1], "the second run must not overwrite");
20137        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20138        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20139        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20140
20141        // Storage failures are evidence too; preserve the command's verdict.
20142        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20143        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20144        let (outcomes, _) = run_commands(
20145            &mut state,
20146            "gate",
20147            "gate",
20148            0,
20149            &["sh".into(), "-c".into()],
20150            &["echo failure-output; exit 7".into()],
20151            tmp.path(),
20152            Duration::from_secs(10),
20153        )
20154        .await;
20155        assert_eq!(outcomes[0].code, Some(7));
20156        assert!(state.events.iter().any(|e| {
20157            e.node == "gate" && e.message.contains("could not save the full gate output:")
20158        }));
20159    }
20160
20161    #[cfg(unix)]
20162    #[tokio::test]
20163    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20164        crate::run::pin_test_home();
20165        let tmp = tempfile::tempdir().unwrap();
20166        let mut config = Config::default();
20167        config.disk.min_free_bytes = 0;
20168        let mut state = RunState::new(
20169            tmp.path().to_owned(),
20170            "main".into(),
20171            "base".into(),
20172            "verification evidence".into(),
20173            config,
20174        );
20175        let (outcomes, pids) = run_commands(
20176            &mut state,
20177            "gate",
20178            "gate",
20179            1,
20180            &["sh".into(), "-c".into()],
20181            &[
20182                "printf before-timeout; exec sleep 30".into(),
20183                "kill -TERM $$".into(),
20184                "exit 7".into(),
20185                "printf success".into(),
20186            ],
20187            tmp.path(),
20188            Duration::from_secs(1),
20189        )
20190        .await;
20191        wait_for_timed_out_children_to_die(&pids).await;
20192        assert_eq!(
20193            pids.len(),
20194            1,
20195            "unleased timeout retains its death-check PID"
20196        );
20197        assert_eq!(outcomes.len(), 4);
20198        assert_eq!(outcomes[0].code, None);
20199        assert!(outcomes[0].output_tail.contains("before-timeout"));
20200        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20201        assert_eq!(outcomes[1].code, None);
20202        assert!(outcomes[1].output_tail.contains("command terminated:"));
20203        assert!(!outcomes[1].output_tail.contains("timed out"));
20204        assert_eq!(outcomes[2].code, Some(7));
20205        assert!(!outcomes[2].output_tail.contains("timed out"));
20206        assert_eq!(outcomes[3].code, Some(0));
20207        assert_eq!(outcomes[3].output_tail, "success");
20208        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20209    }
20210
20211    /// The addendum's second gap: a `verify.gate` command running for real
20212    /// wall-clock time had nothing at all to show for it in `active` before
20213    /// `run_commands` learned to record it — a run could sit in `Gating` for
20214    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20215    /// what was actually happening. Proven with a genuinely still-running
20216    /// command, not just a before/after check on the final state: a poller
20217    /// task reads the same `run.json` `gate()` is writing, the same way the
20218    /// phone or `magi show` would, while the shell command is still blocked
20219    /// on its own release marker.
20220    #[tokio::test]
20221    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20222        crate::run::pin_test_home();
20223
20224        let tmp = tempfile::tempdir().expect("tempdir");
20225        let repo = tmp.path().join("repo");
20226        std::fs::create_dir_all(&repo).unwrap();
20227        init_repo(&repo);
20228
20229        let mut config = Config::default();
20230        config.disk.min_free_bytes = 0;
20231        config.verify.gate = vec![
20232            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20233             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20234                .to_owned(),
20235        ];
20236
20237        let mut state = RunState::new(
20238            repo.clone(),
20239            "main".to_owned(),
20240            "deadbeef".to_owned(),
20241            "task".to_owned(),
20242            config,
20243        );
20244        let run_id = state.id.clone();
20245        state.candidates = vec![Candidate {
20246            index: 0,
20247            label: 'A',
20248            agent: "alpha".to_owned(),
20249            branch: "does-not-exist".to_owned(),
20250            worktree: repo.clone(),
20251            summary: String::new(),
20252            stat: String::new(),
20253            files: 0,
20254            commits: 0,
20255            empty: false,
20256            failed: None,
20257            verified_noop: None,
20258            duration_ms: 0,
20259            folded: false,
20260        }];
20261        state.tally = Some(Tally {
20262            first_choice: BTreeMap::from([('A', 1)]),
20263            borda: BTreeMap::new(),
20264            winner: 'A',
20265            rankings: 1,
20266            unanimous_initial: true,
20267            deliberated: false,
20268            changed_votes: 0,
20269            unanimous_final: true,
20270            tie_break: None,
20271            judges: 0,
20272            present: 0,
20273            quorum: 0,
20274            met_quorum: true,
20275            uncontested: Some("only candidate A produced a change".to_owned()),
20276        });
20277        state.reviews = vec![ReviewRound {
20278            round: 1,
20279            head: "deadbeef".to_owned(),
20280            verified_head: None,
20281            verified_at: None,
20282            reviews: Vec::new(),
20283            e2e: Vec::new(),
20284            fix: None,
20285            blocking: 0,
20286            answered: 0,
20287            expected: 0,
20288            clean: true,
20289            verify_retried: false,
20290            e2e_deferred: false,
20291            e2e_defer_reason: None,
20292            progressed: false,
20293            vote_split: false,
20294            reconsideration: Vec::new(),
20295            verdict: None,
20296        }];
20297
20298        let mut runner = Runner {
20299            state,
20300            roles: ResolvedRoles {
20301                implementers: Vec::new(),
20302                judges: Vec::new(),
20303                reviewers: Vec::new(),
20304                fixer: None,
20305                conductor: conductor(),
20306                implementer_roster: Vec::new(),
20307                judge_roster: Vec::new(),
20308                reviewer_roster: Vec::new(),
20309            },
20310            sem: Arc::new(Semaphore::new(1)),
20311            pause: Pause::new(),
20312            interrupt: Pause::new(),
20313        };
20314
20315        let started_marker = repo.join("started.marker");
20316        let release_marker = repo.join("release.marker");
20317        let poller = tokio::spawn(async move {
20318            // Bounded so a regression that never records the task entry
20319            // fails this test in seconds instead of hanging the suite —
20320            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20321            // does_not_cut_it_short` uses for the same reason.
20322            for _ in 0..100 {
20323                if started_marker.exists()
20324                    && let Ok(s) = crate::run::RunState::load(&run_id)
20325                    && let Some(a) = s.active.get("gate")
20326                {
20327                    std::fs::write(&release_marker, b"go").expect("release marker");
20328                    return Some(a.clone());
20329                }
20330                tokio::time::sleep(Duration::from_millis(50)).await;
20331            }
20332            None
20333        });
20334
20335        runner.gate().await.expect("gate");
20336        let captured = poller.await.expect("poller task");
20337        let captured = captured.expect(
20338            "the poller never saw a `gate` task entry in run.json while the command was \
20339             still blocked on its own release marker",
20340        );
20341
20342        assert_eq!(captured.task.as_deref(), Some("gate"));
20343        assert_eq!(captured.node, "gate");
20344        assert_eq!(captured.index, Some(1));
20345        assert_eq!(captured.total, Some(1));
20346        assert!(
20347            captured
20348                .command
20349                .as_deref()
20350                .is_some_and(|c| c.contains("started.marker")),
20351            "{captured:?}"
20352        );
20353
20354        assert!(
20355            runner.state.active.is_empty(),
20356            "the entry must be cleared once the command actually finished: {:?}",
20357            runner.state.active
20358        );
20359        assert!(runner.state.gate_ran);
20360        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20361    }
20362
20363    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20364    /// record for `land`; one with only a Minor, or no reject, leaves none.
20365    #[tokio::test]
20366    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20367        use crate::verdict::{Finding, ReviewVote, Severity};
20368        crate::run::pin_test_home();
20369        let tmp = tempfile::tempdir().expect("tempdir");
20370        let repo = tmp.path().join("repo");
20371        std::fs::create_dir_all(&repo).unwrap();
20372        init_repo(&repo);
20373
20374        for (severity, vote, expect) in [
20375            (Severity::Major, ReviewVote::Reject, true),
20376            (Severity::Minor, ReviewVote::Reject, false),
20377            (Severity::Major, ReviewVote::Approve, false),
20378        ] {
20379            let mut round = review_round(false, 1, 1, 1, false, true);
20380            round.reviews = vec![ReviewRecord {
20381                reviewer: 1,
20382                agent: "alpha".to_owned(),
20383                summary: String::new(),
20384                findings: vec![Finding {
20385                    id: "R1-1-1".to_owned(),
20386                    severity,
20387                    file: None,
20388                    line: None,
20389                    title: "t".to_owned(),
20390                    detail: String::new(),
20391                }],
20392                vote: Some(vote),
20393                failed: None,
20394                duration_ms: 0,
20395                attempts: 0,
20396            }];
20397            let mut state = RunState::new(
20398                repo.clone(),
20399                "main".to_owned(),
20400                "deadbeef".to_owned(),
20401                "task".to_owned(),
20402                Config::default(),
20403            );
20404            state.reviews = vec![round];
20405            let mut runner = Runner {
20406                state,
20407                roles: ResolvedRoles {
20408                    implementers: Vec::new(),
20409                    judges: Vec::new(),
20410                    reviewers: Vec::new(),
20411                    fixer: None,
20412                    conductor: conductor(),
20413                    implementer_roster: Vec::new(),
20414                    judge_roster: Vec::new(),
20415                    reviewer_roster: Vec::new(),
20416                },
20417                sem: Arc::new(Semaphore::new(1)),
20418                pause: Pause::new(),
20419                interrupt: Pause::new(),
20420            };
20421            let shell = runner.state.config.shell();
20422            runner
20423                .stop_reviewing("round budget spent", &shell, &repo)
20424                .await
20425                .expect("stop_reviewing");
20426            assert_eq!(runner.state.status, RunStatus::Gating);
20427            assert_eq!(
20428                runner.state.contested_handoff.is_some(),
20429                expect,
20430                "{severity:?} + {vote:?}"
20431            );
20432        }
20433    }
20434
20435    /// The shape the incident this whole fix responds to actually had: the
20436    /// round budget spent, the last round's own e2e blocked on the shared
20437    /// build cache (held here by a live pid — this test process — exactly
20438    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20439    /// without forking a process). `stop_reviewing` must retry it — not
20440    /// silently leave the round looking untouched (the catch-up-only half of
20441    /// the bug), and not read the contention as a red `e2e` and block the
20442    /// run on it (the other half). Called directly, the same way
20443    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20444    /// above exercises `gate`, so this never needs a real cargo build to
20445    /// reach: the lease is never released, so `with_cache_lease` never gets
20446    /// past acquiring it into anything that would need a real workspace.
20447    #[tokio::test]
20448    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20449        crate::run::pin_test_home();
20450        let home = crate::run::home();
20451
20452        let tmp = tempfile::tempdir().expect("tempdir");
20453        let repo = tmp.path().join("repo");
20454        std::fs::create_dir_all(&repo).unwrap();
20455        init_repo(&repo);
20456        let head = crate::git::rev_parse(&repo, "HEAD")
20457            .await
20458            .expect("rev-parse");
20459        // Unique to this test, so holding its lease cannot collide with
20460        // another test sharing the same process-wide `home`.
20461        let cache_dir = tmp.path().join("target");
20462
20463        let mut config = Config::default();
20464        config.verify.e2e = vec![format!(
20465            "CARGO_TARGET_DIR='{}' test -f README.md",
20466            cache_dir.display()
20467        )];
20468        config.graph.review_rounds = 1;
20469        // Bounded so a regression that does start waiting fails the test in
20470        // seconds, not hangs it.
20471        config.graph.timeout_verify = Some(2);
20472        config.graph.timeout_resource_wait = 2;
20473
20474        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20475        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20476            .expect("no io error acquiring directly")
20477        {
20478            crate::cache::AcquireOutcome::Acquired(g) => g,
20479            crate::cache::AcquireOutcome::Busy(b) => {
20480                panic!("expected the direct acquire to win the lease first: {b:?}")
20481            }
20482        };
20483
20484        let mut state = RunState::new(
20485            repo.clone(),
20486            "main".to_owned(),
20487            head.clone(),
20488            "task".to_owned(),
20489            config,
20490        );
20491        state.candidates = vec![Candidate {
20492            index: 0,
20493            label: 'A',
20494            agent: "alpha".to_owned(),
20495            branch: "does-not-exist".to_owned(),
20496            worktree: repo.clone(),
20497            summary: String::new(),
20498            stat: String::new(),
20499            files: 0,
20500            commits: 0,
20501            empty: false,
20502            failed: None,
20503            verified_noop: None,
20504            duration_ms: 0,
20505            folded: false,
20506        }];
20507        state.tally = Some(Tally {
20508            first_choice: BTreeMap::from([('A', 1)]),
20509            borda: BTreeMap::new(),
20510            winner: 'A',
20511            rankings: 1,
20512            unanimous_initial: true,
20513            deliberated: false,
20514            changed_votes: 0,
20515            unanimous_final: true,
20516            tie_break: None,
20517            judges: 0,
20518            present: 0,
20519            quorum: 0,
20520            met_quorum: true,
20521            uncontested: Some("only candidate A produced a change".to_owned()),
20522        });
20523        // The round budget's last round, deferred: `needs_catchup_run`'s
20524        // other trigger. `stop_reviewing`'s retry machinery must treat this
20525        // exactly like a resource-blocked attempt once it actually runs.
20526        state.reviews = vec![ReviewRound {
20527            round: 1,
20528            head: head.clone(),
20529            verified_head: None,
20530            verified_at: None,
20531            reviews: Vec::new(),
20532            e2e: Vec::new(),
20533            fix: None,
20534            blocking: 1,
20535            answered: 1,
20536            expected: 1,
20537            clean: false,
20538            verify_retried: false,
20539            e2e_deferred: true,
20540            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20541            progressed: false,
20542            vote_split: false,
20543            reconsideration: Vec::new(),
20544            verdict: None,
20545        }];
20546
20547        let mut runner = Runner {
20548            state,
20549            roles: ResolvedRoles {
20550                implementers: Vec::new(),
20551                judges: Vec::new(),
20552                reviewers: Vec::new(),
20553                fixer: None,
20554                conductor: conductor(),
20555                implementer_roster: Vec::new(),
20556                judge_roster: Vec::new(),
20557                reviewer_roster: Vec::new(),
20558            },
20559            sem: Arc::new(Semaphore::new(1)),
20560            pause: Pause::new(),
20561            interrupt: Pause::new(),
20562        };
20563
20564        let shell = runner.state.config.shell();
20565        runner
20566            .stop_reviewing("round budget spent", &shell, &repo)
20567            .await
20568            .expect("stop_reviewing");
20569
20570        let last = runner.state.reviews.last().expect("round record");
20571        assert_eq!(
20572            last.e2e_status(),
20573            E2eStatus::ResourceBlocked,
20574            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20575             failed: {last:?}"
20576        );
20577        assert_eq!(
20578            last.verified_head.as_deref(),
20579            Some(head.as_str()),
20580            "which commit this attempt targeted is known even though nothing finished checking \
20581             it"
20582        );
20583        let first_attempt_at = last
20584            .verified_at
20585            .expect("when this attempt ran is known too");
20586        assert_ne!(
20587            runner.state.status,
20588            RunStatus::Blocked,
20589            "contention is evidence about the machine, not the patch — it must not settle the \
20590             run as blocked: {:?}",
20591            runner.state.status
20592        );
20593        assert!(
20594            !runner
20595                .state
20596                .events
20597                .iter()
20598                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20599            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20600            runner.state.events
20601        );
20602
20603        // The cache is still held: a later reentry must retry the same
20604        // round's verification again — not leave it looking exactly as
20605        // untouched as the first blocked attempt, which is indistinguishable
20606        // from never having tried again at all.
20607        runner
20608            .stop_reviewing("round budget spent", &shell, &repo)
20609            .await
20610            .expect("stop_reviewing retry");
20611        assert_eq!(
20612            runner.state.reviews.len(),
20613            1,
20614            "no new round was started: {:?}",
20615            runner.state.reviews
20616        );
20617        let last = runner.state.reviews.last().expect("round record");
20618        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20619        assert!(
20620            last.verified_at.expect("still known") > first_attempt_at,
20621            "a second reentry must be a fresh attempt, not a stale copy of the first"
20622        );
20623        assert_ne!(runner.state.status, RunStatus::Blocked);
20624
20625        held.release();
20626    }
20627
20628    #[cfg(unix)]
20629    #[tokio::test]
20630    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20631        ask_test_home();
20632        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20633            let tmp = tempfile::tempdir().unwrap();
20634            let repo = tmp.path().join("repo");
20635            std::fs::create_dir(&repo).unwrap();
20636            init_repo(&repo);
20637            let run_git = |args: &[&str]| {
20638                let out = std::process::Command::new("git")
20639                    .args(args)
20640                    .current_dir(&repo)
20641                    .output()
20642                    .unwrap();
20643                assert!(
20644                    out.status.success(),
20645                    "{args:?}: {}",
20646                    String::from_utf8_lossy(&out.stderr)
20647                );
20648            };
20649            let origin = tmp.path().join("origin.git");
20650            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20651            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20652            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20653            run_git(&["add", "spec.rs"]);
20654            run_git(&["commit", "-m", "initial spec"]);
20655            run_git(&["push", "origin", "main"]);
20656            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20657            run_git(&["checkout", "-b", "fix/review"]);
20658            std::fs::write(
20659                repo.join("check.rs"),
20660                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20661            )
20662            .unwrap();
20663            run_git(&["add", "check.rs"]);
20664            run_git(&["commit", "-m", "use spec"]);
20665            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20666            run_git(&["checkout", "main"]);
20667            std::fs::write(
20668                repo.join("spec.rs"),
20669                "struct AgentSpec { max_turn_secs: u64 }\n",
20670            )
20671            .unwrap();
20672            run_git(&["add", "spec.rs"]);
20673            run_git(&["commit", "-m", "extend spec"]);
20674            run_git(&["push", "origin", "main"]);
20675            run_git(&["checkout", "fix/review"]);
20676
20677            let mut runner = runner_at(RunStatus::Reviewing);
20678            runner.state.repo = repo.clone();
20679            runner.state.base_commit = base;
20680            runner.state.pr_head = Some("fix/review".into());
20681            runner.state.config.graph.review_rounds = max_rounds;
20682            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20683            runner.state.config.graph.sessions = false;
20684            runner.state.config.disk.min_free_bytes = 0;
20685            runner.state.config.verify.e2e =
20686                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20687            runner.state.candidates = vec![Candidate {
20688                index: 0,
20689                label: 'A',
20690                agent: "fixture".into(),
20691                branch: "fix/review".into(),
20692                worktree: repo.clone(),
20693                summary: String::new(),
20694                stat: String::new(),
20695                files: 1,
20696                commits: 1,
20697                empty: false,
20698                failed: None,
20699                verified_noop: None,
20700                duration_ms: 0,
20701                folded: false,
20702            }];
20703            runner.state.tally = Some(Tally {
20704                first_choice: BTreeMap::new(),
20705                borda: BTreeMap::new(),
20706                winner: 'A',
20707                rankings: 0,
20708                unanimous_initial: true,
20709                deliberated: false,
20710                changed_votes: 0,
20711                unanimous_final: true,
20712                tie_break: None,
20713                judges: 0,
20714                present: 0,
20715                quorum: 0,
20716                met_quorum: true,
20717                uncontested: Some("review-only".into()),
20718            });
20719            let mut last = review_round(false, 0, 1, 1, false, false);
20720            last.head = reviewed.clone();
20721            last.e2e_deferred = true;
20722            last.e2e[0].resource_blocked = true;
20723            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20724            if max_rounds == 1 {
20725                last.e2e.clear();
20726            }
20727            runner.state.reviews = vec![last];
20728            let calls = tmp.path().join("fix-calls");
20729            let mut fixer = spec("repair");
20730            fixer
20731                .env
20732                .insert("FIX_CALLS".into(), calls.display().to_string());
20733            fixer.command = vec!["sh".into(), "-c".into(), r#"
20734                echo fix >> "$FIX_CALLS"
20735                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20736                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20737                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20738                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20739            "#.into()];
20740            let mut reviewer = spec("reviewer");
20741            reviewer.command = vec![
20742                "sh".into(),
20743                "-c".into(),
20744                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20745                    .into(),
20746            ];
20747            runner.roles.fixer = Some(vec![fixer]);
20748            runner.roles.reviewers = vec![reviewer];
20749
20750            runner.sync_to_base().await.unwrap();
20751            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20752            assert_ne!(rebased, reviewed);
20753            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20754            if reentry {
20755                // Simulate a crash after catch-up evidence was saved but before
20756                // dispatching the fixer. No dedicated pending-repair field exists.
20757                let shell = runner.state.config.shell();
20758                runner
20759                    .stop_unresolved_review("deferred verification", &shell, &repo)
20760                    .await
20761                    .unwrap();
20762                assert!(runner.catchup_repairable());
20763                assert!(!calls.exists());
20764                runner.state = RunState::load(&runner.state.id).unwrap();
20765                assert!(runner.catchup_repairable());
20766            }
20767            runner.review_loop().await.unwrap();
20768            let first = &runner.state.reviews[0];
20769            assert_eq!(first.head, reviewed);
20770            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20771            assert!(
20772                first.verify_retried,
20773                "a reproduced build failure gets exactly one retry: {:?}",
20774                first.e2e
20775            );
20776            assert!(first.e2e[0].output_tail.contains("E0063"));
20777            assert!(!runner.catchup_repairable());
20778            if max_rounds == 2 {
20779                assert_eq!(
20780                    runner.state.status,
20781                    RunStatus::Gating,
20782                    "{}",
20783                    crate::report::run(&runner.state)
20784                );
20785                assert_eq!(runner.state.reviews.len(), 2);
20786                assert!(first.fix.as_ref().unwrap().committed);
20787                assert!(runner.state.reviews[1].clean);
20788                assert_eq!(
20789                    runner
20790                        .state
20791                        .reviews
20792                        .iter()
20793                        .filter(|r| r.fix.is_some())
20794                        .count(),
20795                    1
20796                );
20797                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20798                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20799            } else {
20800                assert_eq!(runner.state.status, RunStatus::Blocked);
20801                assert!(first.fix.is_none());
20802                assert!(!calls.exists());
20803                let recovery = crate::recovery::assess(
20804                    "prior",
20805                    Some(&runner.state),
20806                    &runner.landing_base(),
20807                    crate::recovery::Relation::Diverged,
20808                );
20809                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20810                let reasons = recovery.reasons.join("\n");
20811                assert!(
20812                    reasons.contains(&reviewed) && reasons.contains(&rebased),
20813                    "{reasons}"
20814                );
20815                assert!(
20816                    reasons.contains("base sync to")
20817                        && reasons.contains("additional review is required"),
20818                    "{reasons}"
20819                );
20820            }
20821        }
20822    }
20823
20824    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
20825    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
20826    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
20827    /// own top-of-function fast path (`review_conclusion`) correctly reads
20828    /// this shape as `None` rather than guessing `Blocked`, and the loop's
20829    /// own `for` range is empty once the round budget is spent, so
20830    /// `review_loop` must retry the check itself rather than silently doing
20831    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
20832    /// above exercises directly, but through `review_loop`'s own entry point
20833    /// this time, proving the wiring between the two rather than just the
20834    /// retry logic in isolation.
20835    #[tokio::test]
20836    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20837        crate::run::pin_test_home();
20838        let home = crate::run::home();
20839
20840        let tmp = tempfile::tempdir().expect("tempdir");
20841        let repo = tmp.path().join("repo");
20842        std::fs::create_dir_all(&repo).unwrap();
20843        init_repo(&repo);
20844        let head = crate::git::rev_parse(&repo, "HEAD")
20845            .await
20846            .expect("rev-parse");
20847        let cache_dir = tmp.path().join("target");
20848
20849        let mut config = Config::default();
20850        config.verify.e2e = vec![format!(
20851            "CARGO_TARGET_DIR='{}' test -f README.md",
20852            cache_dir.display()
20853        )];
20854        config.graph.review_rounds = 1;
20855        config.graph.timeout_verify = Some(2);
20856        config.graph.timeout_resource_wait = 2;
20857
20858        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20859        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20860            .expect("no io error acquiring directly")
20861        {
20862            crate::cache::AcquireOutcome::Acquired(g) => g,
20863            crate::cache::AcquireOutcome::Busy(b) => {
20864                panic!("expected the direct acquire to win the lease first: {b:?}")
20865            }
20866        };
20867
20868        let mut state = RunState::new(
20869            repo.clone(),
20870            "main".to_owned(),
20871            head.clone(),
20872            "task".to_owned(),
20873            config,
20874        );
20875        state.candidates = vec![Candidate {
20876            index: 0,
20877            label: 'A',
20878            agent: "alpha".to_owned(),
20879            branch: "does-not-exist".to_owned(),
20880            worktree: repo.clone(),
20881            summary: String::new(),
20882            stat: String::new(),
20883            files: 0,
20884            commits: 0,
20885            empty: false,
20886            failed: None,
20887            verified_noop: None,
20888            duration_ms: 0,
20889            folded: false,
20890        }];
20891        state.tally = Some(Tally {
20892            first_choice: BTreeMap::from([('A', 1)]),
20893            borda: BTreeMap::new(),
20894            winner: 'A',
20895            rankings: 1,
20896            unanimous_initial: true,
20897            deliberated: false,
20898            changed_votes: 0,
20899            unanimous_final: true,
20900            tie_break: None,
20901            judges: 0,
20902            present: 0,
20903            quorum: 0,
20904            met_quorum: true,
20905            uncontested: Some("only candidate A produced a change".to_owned()),
20906        });
20907        // The exact shape a prior process's `stop_reviewing` would have left
20908        // on disk: the round budget's last round, a real attempt already
20909        // made and already resource-blocked.
20910        state.reviews = vec![ReviewRound {
20911            round: 1,
20912            head: head.clone(),
20913            verified_head: Some(head.clone()),
20914            verified_at: Some(jiff::Timestamp::now()),
20915            reviews: Vec::new(),
20916            e2e: vec![CommandOutcome {
20917                command: format!(
20918                    "CARGO_TARGET_DIR='{}' test -f README.md",
20919                    cache_dir.display()
20920                ),
20921                code: None,
20922                output_tail: "waiting for the shared build cache".to_owned(),
20923                duration_ms: 0,
20924                resource_blocked: true,
20925            }],
20926            fix: None,
20927            blocking: 1,
20928            answered: 1,
20929            expected: 1,
20930            clean: false,
20931            verify_retried: false,
20932            e2e_deferred: false,
20933            e2e_defer_reason: None,
20934            progressed: false,
20935            vote_split: false,
20936            reconsideration: Vec::new(),
20937            verdict: None,
20938        }];
20939
20940        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20941        let mut runner = Runner {
20942            state,
20943            roles: ResolvedRoles {
20944                implementers: Vec::new(),
20945                judges: Vec::new(),
20946                reviewers: Vec::new(),
20947                fixer: None,
20948                conductor: conductor(),
20949                implementer_roster: Vec::new(),
20950                judge_roster: Vec::new(),
20951                reviewer_roster: Vec::new(),
20952            },
20953            sem: Arc::new(Semaphore::new(1)),
20954            pause: Pause::new(),
20955            interrupt: Pause::new(),
20956        };
20957
20958        // The lease is still held throughout, so this reentry's own retry is
20959        // also contended — proving `review_loop` actually tried again (not
20960        // that it happened to succeed) is what the timestamp comparison
20961        // below is for.
20962        runner.review_loop().await.expect("review_loop");
20963
20964        assert_eq!(
20965            runner.state.reviews.len(),
20966            1,
20967            "no new round was started on top of the unresolved one: {:?}",
20968            runner.state.reviews
20969        );
20970        let last = &runner.state.reviews[0];
20971        assert_eq!(
20972            last.e2e_status(),
20973            E2eStatus::ResourceBlocked,
20974            "still contended: {last:?}"
20975        );
20976        assert!(
20977            last.verified_at.expect("still known") > first_attempt_at,
20978            "review_loop must have actually retried the check, not left it exactly as found"
20979        );
20980        assert_ne!(
20981            runner.state.status,
20982            RunStatus::Blocked,
20983            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20984            runner.state.status
20985        );
20986
20987        held.release();
20988    }
20989
20990    #[tokio::test]
20991    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
20992        crate::run::pin_test_home();
20993        for status in [RunStatus::Blocked, RunStatus::Gating] {
20994            let mut runner = runner_at(status);
20995            runner.state.config.graph.land = true;
20996            runner.state.gate_ran = true;
20997            runner.state.merge = Some(MergeOutcome {
20998                mode: MergeMode::Pr,
20999                ok: false,
21000                detail: "the fixer produced no commit".into(),
21001                empty: false,
21002            });
21003            let url = "https://example.invalid/o/r/pull/1";
21004            runner.state.pr = Some(crate::run::PrRecord {
21005                url: url.into(),
21006                number: 1,
21007                state: "open".into(),
21008                checks: "red".into(),
21009                round: 1,
21010                rounds: 1,
21011                red_at_merge: Vec::new(),
21012            });
21013            runner.state.save().unwrap();
21014            // The nonexistent repository makes observation fail before a forge
21015            // command can run. We still must reach land, rather than silently
21016            // return from the graph with its old gate and merge records.
21017            runner.execute_graph().await.unwrap();
21018            assert_eq!(runner.state.status, RunStatus::Blocked);
21019            assert!(runner.state.land_sync_pending);
21020            assert!(runner.state.events.iter().any(|event|
21021                event.node == "land" && event.message == format!("watching {url}")));
21022            assert_eq!(
21023                runner.state.merge.as_ref().unwrap().detail,
21024                "the fixer produced no commit"
21025            );
21026        }
21027    }
21028
21029    #[tokio::test]
21030    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21031        crate::run::pin_test_home();
21032        let tmp = tempfile::tempdir().expect("tempdir");
21033        let repo = tmp.path().join("repo");
21034        std::fs::create_dir_all(&repo).unwrap();
21035        init_repo(&repo);
21036
21037        let mut config = Config::default();
21038        config.merge.mode = MergeMode::Pr;
21039        config.graph.land = true;
21040        config.graph.land_approval = false;
21041
21042        let mut state = RunState::new(
21043            repo.clone(),
21044            "main".to_owned(),
21045            "deadbeef".to_owned(),
21046            "task".to_owned(),
21047            config,
21048        );
21049        state.candidates = vec![Candidate {
21050            index: 0,
21051            label: 'A',
21052            agent: "alpha".to_owned(),
21053            branch: "does-not-exist".to_owned(),
21054            worktree: repo.clone(),
21055            summary: String::new(),
21056            stat: String::new(),
21057            files: 0,
21058            commits: 0,
21059            empty: false,
21060            failed: None,
21061            verified_noop: None,
21062            duration_ms: 0,
21063            folded: false,
21064        }];
21065        state.tally = Some(Tally {
21066            first_choice: BTreeMap::from([('A', 1)]),
21067            borda: BTreeMap::new(),
21068            winner: 'A',
21069            rankings: 1,
21070            unanimous_initial: true,
21071            deliberated: false,
21072            changed_votes: 0,
21073            unanimous_final: true,
21074            tie_break: None,
21075            judges: 0,
21076            present: 0,
21077            quorum: 0,
21078            met_quorum: true,
21079            uncontested: Some("only candidate A produced a change".to_owned()),
21080        });
21081        state.reviews = vec![ReviewRound {
21082            round: 1,
21083            head: "deadbeef".to_owned(),
21084            verified_head: None,
21085            verified_at: None,
21086            reviews: Vec::new(),
21087            e2e: Vec::new(),
21088            fix: None,
21089            blocking: 0,
21090            answered: 0,
21091            expected: 0,
21092            clean: true,
21093            verify_retried: false,
21094            e2e_deferred: false,
21095            e2e_defer_reason: None,
21096            progressed: false,
21097            vote_split: false,
21098            reconsideration: Vec::new(),
21099            verdict: None,
21100        }];
21101        state.gate = vec![CommandOutcome {
21102            command: "test".to_owned(),
21103            code: Some(0),
21104            output_tail: String::new(),
21105            duration_ms: 0,
21106            resource_blocked: false,
21107        }];
21108        state.gate_ran = true;
21109        // A first pass through `merge` already pushed and opened this pull
21110        // request; `status` is `Landing` because a previous call into `land`
21111        // parked or was interrupted before it reached a terminal outcome.
21112        state.status = RunStatus::Landing;
21113        state.merge = Some(MergeOutcome {
21114            mode: MergeMode::Pr,
21115            ok: true,
21116            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21117            empty: false,
21118        });
21119
21120        // The Landing-resume shortcut calls `run_land` directly rather than
21121        // through `merge`, which is exactly the call site that used to skip
21122        // `settle_questions` - see the fixture below.
21123        ask_test_home();
21124        // A real resume loads a persisted run; driver ownership is checked
21125        // against that record before the Landing shortcut is entered.
21126        state.save().expect("save landing run");
21127        let store = ask::Questions::open();
21128        let q = ask_open_question(&store, &state.id);
21129
21130        let mut runner = Runner {
21131            state,
21132            roles: ResolvedRoles {
21133                implementers: Vec::new(),
21134                judges: Vec::new(),
21135                reviewers: Vec::new(),
21136                fixer: None,
21137                conductor: conductor(),
21138                implementer_roster: Vec::new(),
21139                judge_roster: Vec::new(),
21140                reviewer_roster: Vec::new(),
21141            },
21142            sem: Arc::new(Semaphore::new(1)),
21143            pause: Pause::new(),
21144            interrupt: Pause::new(),
21145        };
21146
21147        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21148        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21149        // exactly because `review_loop` would otherwise clobber the marker
21150        // first.
21151        runner.execute().await.expect("execute");
21152
21153        assert_eq!(
21154            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21155            Some("https://example.invalid/x/y/pull/1"),
21156            "reentry must not push again or open a second pull request over the \
21157             one `land` is already watching"
21158        );
21159        assert_ne!(
21160            runner.state.status,
21161            RunStatus::Landing,
21162            "land could not actually reach the fake pull request, so it must \
21163             have given up rather than left the run silently parked forever"
21164        );
21165        // `land` could not reach the fake pull request, so it gave up into
21166        // `Blocked` - still resumable, so the question must not have been
21167        // swept just because this branch now also calls `settle_questions`.
21168        assert_eq!(runner.state.status, RunStatus::Blocked);
21169        assert!(
21170            store.get(&q.id).unwrap().status.open(),
21171            "Blocked is still alive; settle_questions must have been a no-op here"
21172        );
21173    }
21174
21175    fn state_with_round(round: ReviewRound) -> RunState {
21176        let mut s = RunState::new(
21177            PathBuf::from("/repo"),
21178            "main".to_owned(),
21179            "abc1234".to_owned(),
21180            "add retries".to_owned(),
21181            Config::default(),
21182        );
21183        s.reviews = vec![round];
21184        s
21185    }
21186
21187    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21188        crate::verdict::Finding {
21189            id: id.to_owned(),
21190            severity,
21191            file: None,
21192            line: None,
21193            title: title.to_owned(),
21194            detail: String::new(),
21195        }
21196    }
21197
21198    #[test]
21199    fn pr_body_names_open_findings_and_declined_ones() {
21200        let round = ReviewRound {
21201            round: 2,
21202            head: "deadbee".to_owned(),
21203            verified_head: None,
21204            verified_at: None,
21205            reviews: vec![ReviewRecord {
21206                attempts: 0,
21207                reviewer: 1,
21208                agent: "alpha".to_owned(),
21209                summary: String::new(),
21210                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21211                vote: None,
21212                failed: None,
21213                duration_ms: 0,
21214            }],
21215            e2e: vec![CommandOutcome {
21216                command: "cargo test".to_owned(),
21217                code: Some(0),
21218                output_tail: String::new(),
21219                duration_ms: 0,
21220                resource_blocked: false,
21221            }],
21222            verify_retried: false,
21223            e2e_deferred: false,
21224            e2e_defer_reason: None,
21225            fix: Some(FixRecord {
21226                agent: "alpha".to_owned(),
21227                addressed: Vec::new(),
21228                rejected: vec![crate::verdict::Rejection {
21229                    id: "R1-1-1".to_owned(),
21230                    why: "not reachable from any caller".to_owned(),
21231                }],
21232                notes: String::new(),
21233                committed: true,
21234                failed: None,
21235                duration_ms: 0,
21236                continuation: None,
21237            }),
21238            blocking: 0,
21239            answered: 1,
21240            expected: 1,
21241            clean: false,
21242            progressed: true,
21243            vote_split: false,
21244            reconsideration: Vec::new(),
21245            verdict: None,
21246        };
21247        let state = state_with_round(round);
21248        let body = pr_message(&state, 'A').body;
21249
21250        assert!(body.contains("add retries"), "the task must still be there");
21251        assert!(body.contains("R2-1-1"), "{body}");
21252        assert!(!body.contains("unused import"), "{body}");
21253        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21254        assert!(
21255            !body.contains("not reachable from any caller"),
21256            "the reason it was declined: {body}"
21257        );
21258    }
21259
21260    #[test]
21261    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21262        let round = ReviewRound {
21263            round: 1,
21264            head: "deadbee".to_owned(),
21265            verified_head: None,
21266            verified_at: None,
21267            reviews: vec![ReviewRecord {
21268                attempts: 0,
21269                reviewer: 1,
21270                agent: "alpha".to_owned(),
21271                summary: String::new(),
21272                findings: Vec::new(),
21273                vote: None,
21274                failed: None,
21275                duration_ms: 0,
21276            }],
21277            e2e: Vec::new(),
21278            verify_retried: false,
21279            e2e_deferred: false,
21280            e2e_defer_reason: None,
21281            fix: None,
21282            blocking: 0,
21283            answered: 1,
21284            expected: 1,
21285            clean: true,
21286            progressed: false,
21287            vote_split: false,
21288            reconsideration: Vec::new(),
21289            verdict: None,
21290        };
21291        let state = state_with_round(round);
21292        let body = pr_message(&state, 'A').body;
21293        assert!(!body.contains("Open review findings"), "{body}");
21294        assert!(!body.contains("Declined"), "{body}");
21295    }
21296
21297    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21298        let mut state = RunState::new(
21299            PathBuf::from("/repo"),
21300            "main".to_owned(),
21301            "abc1234".to_owned(),
21302            instruction.to_owned(),
21303            Config::default(),
21304        );
21305        // These fixtures test execution flow, independent of host disk capacity.
21306        state.config.disk.min_free_bytes = 0;
21307        state.candidates.push(Candidate {
21308            index: 0,
21309            label: 'A',
21310            agent: "alpha".to_owned(),
21311            branch: "magi/x/A".to_owned(),
21312            worktree: PathBuf::from("/wt"),
21313            summary: summary.to_owned(),
21314            stat: String::new(),
21315            files: 1,
21316            commits: 1,
21317            empty: false,
21318            failed: None,
21319            verified_noop: None,
21320            folded: false,
21321            duration_ms: 0,
21322        });
21323        state
21324    }
21325
21326    fn review_state(subjects: &[&str]) -> RunState {
21327        let mut state = state_with_summary(
21328            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21329            "",
21330        );
21331        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21332        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21333        state
21334    }
21335
21336    /// A branch rebased onto a main that moved past the recorded
21337    /// `base_commit` is titled from its own first commit, never main's.
21338    #[tokio::test]
21339    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21340        ask_test_home();
21341        let tmp = tempfile::tempdir().unwrap();
21342        let repo = tmp.path().join("repo");
21343        std::fs::create_dir_all(&repo).unwrap();
21344        init_repo(&repo);
21345        let origin = tmp.path().join("origin.git");
21346        let g = |dir: &Path, args: &[&str]| {
21347            let out = std::process::Command::new("git")
21348                .args(args)
21349                .current_dir(dir)
21350                .quiet()
21351                .output()
21352                .expect("spawn git");
21353            assert!(
21354                out.status.success(),
21355                "git {args:?}: {}",
21356                String::from_utf8_lossy(&out.stderr)
21357            );
21358        };
21359        g(
21360            tmp.path(),
21361            &[
21362                "clone",
21363                "--bare",
21364                "-q",
21365                repo.to_str().unwrap(),
21366                origin.to_str().unwrap(),
21367            ],
21368        );
21369        g(
21370            &repo,
21371            &["remote", "add", "origin", origin.to_str().unwrap()],
21372        );
21373        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21374
21375        // Main moves on; the branch is built on top of the new main.
21376        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21377        g(&repo, &["add", "-A"]);
21378        g(
21379            &repo,
21380            &["commit", "-q", "-m", "chore(deps): update a crate"],
21381        );
21382        g(&repo, &["push", "-q", "origin", "main"]);
21383        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21384        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21385        g(&repo, &["add", "-A"]);
21386        g(
21387            &repo,
21388            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21389        );
21390        g(&repo, &["checkout", "-q", "main"]);
21391
21392        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21393        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21394        // Without a readable tracking ref the recorded base's merge base is used.
21395        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21396        assert_eq!(fallback, c1);
21397
21398        let mut state = review_state(&[
21399            "chore(deps): update a crate",
21400            "fix(daemon): apply a chosen action",
21401        ]);
21402        state.repo = repo.clone();
21403        state.base_branch = "main".to_owned();
21404        state.base_commit = c1;
21405        refresh_reviewed_commits(&mut state, "feat/own").await;
21406        assert_eq!(
21407            state.reviewed_commits,
21408            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21409        );
21410        assert_eq!(
21411            review_title(&state).as_deref(),
21412            Some("fix(daemon): apply a chosen action")
21413        );
21414        assert_eq!(
21415            leaked_subjects(&state, "feat/own").await,
21416            Some(vec!["chore(deps): update a crate".to_owned()])
21417        );
21418        // A stale tracking ref is still used when the fetch fails, but the
21419        // leak list is withheld.
21420        state.config.merge.remote = "nowhere".to_owned();
21421        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21422    }
21423
21424    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21425    /// with `origin/main` one commit behind it, and a summarizing agent that
21426    /// prints whatever `reply` holds.
21427    struct SummaryFixture {
21428        _tmp: tempfile::TempDir,
21429        repo: PathBuf,
21430        reply: PathBuf,
21431        runner: Runner,
21432    }
21433
21434    fn summary_fixture() -> SummaryFixture {
21435        ask_test_home();
21436        let tmp = tempfile::tempdir().unwrap();
21437        let repo = tmp.path().join("repo");
21438        std::fs::create_dir_all(&repo).unwrap();
21439        init_repo(&repo);
21440        let origin = tmp.path().join("origin.git");
21441        let g = |args: &[&str]| {
21442            let out = std::process::Command::new("git")
21443                .args(args)
21444                .current_dir(&repo)
21445                .quiet()
21446                .output()
21447                .expect("spawn git");
21448            assert!(
21449                out.status.success(),
21450                "git {args:?}: {}",
21451                String::from_utf8_lossy(&out.stderr)
21452            );
21453        };
21454        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21455        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21456        g(&["push", "-q", "origin", "main"]);
21457        g(&["checkout", "-q", "-b", "feat/own"]);
21458        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21459        g(&["add", "-A"]);
21460        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21461        let reply = tmp.path().join("reply.txt");
21462        std::fs::write(
21463            &reply,
21464            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21465        )
21466        .unwrap();
21467        let mut runner = runner_at(RunStatus::Gating);
21468        let mut state = review_state(&["feat(own): add the own file"]);
21469        state.repo = repo.clone();
21470        state.base_branch = "main".to_owned();
21471        state.candidates[0].branch = "feat/own".to_owned();
21472        state.candidates[0].worktree = repo.clone();
21473        let mut author = spec("alpha");
21474        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21475        author
21476            .env
21477            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21478        state.config.agents = vec![author];
21479        runner.state = state;
21480        SummaryFixture {
21481            _tmp: tmp,
21482            repo,
21483            reply,
21484            runner,
21485        }
21486    }
21487
21488    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21489        let winner = f.runner.state.candidates[0].clone();
21490        f.runner
21491            .guarded_pr_message(&winner, None, true)
21492            .await
21493            .unwrap()
21494            .expect("an English description never parks")
21495    }
21496
21497    fn clean_round_with_minor_findings() -> ReviewRound {
21498        let mut round = passing_round(2, None);
21499        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21500        round.reviews = vec![ReviewRecord {
21501            attempts: 0,
21502            reviewer: 1,
21503            agent: "alpha".into(),
21504            summary: "内部レビュー".into(),
21505            findings: vec![
21506                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21507                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21508            ],
21509            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21510            failed: None,
21511            duration_ms: 0,
21512        }];
21513        round
21514    }
21515
21516    #[tokio::test]
21517    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21518        for review_only in [false, true] {
21519            let mut f = summary_fixture();
21520            if !review_only {
21521                f.runner.state.candidates[0].agent = "alpha".into();
21522            }
21523            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21524            assert!(f.runner.state.reviews[0].clean);
21525            assert!(f.runner.state.reviews[0].fix.is_none());
21526            assert!(f.runner.state.open_findings().is_empty());
21527            assert_eq!(
21528                crate::pr_outcome::required_finding_ids(&f.runner.state),
21529                ["R2-1-1", "R2-1-2"]
21530            );
21531            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21532            let evidence = crate::pr_outcome::summary_evidence(
21533                &f.runner.state,
21534                crate::pr_outcome::FinalHead::At(&head),
21535            );
21536            assert!(
21537                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21538            );
21539            // The ordinary reply omits both IDs: no cached description is adopted.
21540            let omitted = posted(&mut f).await;
21541            assert!(omitted.body.contains("No description"));
21542            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21543            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21544            assert!(f.runner.state.change_summary.is_none());
21545            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21546            let described = posted(&mut f).await;
21547            assert!(described.body.contains("improve documentation for callers"));
21548            assert!(described.body.contains("remaining edge case"));
21549            assert!(described.body.contains("## Open review findings"));
21550            assert!(!described.body.contains("未修正"));
21551        }
21552    }
21553
21554    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21555        let mut f = summary_fixture();
21556        if !review_only {
21557            f.runner.state.candidates[0].agent = "alpha".into();
21558        }
21559        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21560        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21561        fixed
21562            .fix
21563            .as_mut()
21564            .unwrap()
21565            .rejected
21566            .push(crate::verdict::Rejection {
21567                id: "R1-1-1".into(),
21568                why: "呼び出されないコードのため辞退".into(),
21569            });
21570        let mut clean = clean_round_with_minor_findings();
21571        clean.verified_head = Some(head.clone());
21572        f.runner.state.reviews = vec![fixed, clean];
21573        ok_gate(&mut f.runner.state, Some(&head));
21574        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21575        f.runner.state.config.agents.push(scripted("beta", reply));
21576        f.runner.state.config.roles.language_judge =
21577            Some(crate::config::AgentChoice::One("beta".into()));
21578        f
21579    }
21580
21581    #[tokio::test]
21582    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21583        for review_only in [false, true] {
21584            let mut f = rewrite_fixture_with_findings(review_only,
21585                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21586            let m = posted(&mut f).await;
21587            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21588            let expected =
21589                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21590            assert!(m.body.contains(&expected));
21591            assert!(m.body.contains("## Review rounds"));
21592            assert!(
21593                m.body
21594                    .contains("Gate (`verify.gate`): passed on the final commit")
21595            );
21596            assert!(
21597                m.body
21598                    .contains("Tests (`verify.e2e`): passed on the final commit")
21599            );
21600            let c = f.runner.state.change_summary.as_ref().unwrap();
21601            assert!(
21602                m.body
21603                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21604            );
21605            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21606            assert!(crate::pr_outcome::covers_required_findings(
21607                &f.runner.state,
21608                &m.body
21609            ));
21610            assert!(m.body.contains("documentation remains incomplete"));
21611            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21612            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21613            assert!(f.runner.state.github_text.is_none());
21614        }
21615    }
21616
21617    #[tokio::test]
21618    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21619        for review_only in [false, true] {
21620            for reply in [
21621                "TITLE: fix: public description\nA short English description.",
21622                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21623            ] {
21624                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21625                let winner = f.runner.state.candidates[0].clone();
21626                let posted = f
21627                    .runner
21628                    .guarded_pr_message(&winner, None, true)
21629                    .await
21630                    .unwrap();
21631                assert!(posted.is_none());
21632                assert!(f.runner.state.parked);
21633                let withheld = f.runner.state.github_text.as_ref().unwrap();
21634                assert!(
21635                    withheld.rewrite.is_none(),
21636                    "an incomplete rewrite must not be selectable"
21637                );
21638                let q = ask::Questions::open()
21639                    .get(withheld.question.as_ref().unwrap())
21640                    .unwrap();
21641                assert!(
21642                    !q.choices
21643                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21644                );
21645                assert!(
21646                    q.detail.contains("R1-1-1")
21647                        && q.detail.contains("R2-1-1")
21648                        && q.detail.contains("R2-1-2")
21649                );
21650            }
21651        }
21652    }
21653
21654    #[tokio::test]
21655    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21656        for review_only in [false, true] {
21657            let mut f = summary_fixture();
21658            if !review_only {
21659                f.runner.state.candidates[0].agent = "alpha".into();
21660            }
21661            f.runner.state.instruction = "内部タスクの説明".into();
21662            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21663            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21664            f.runner.state.reviews = vec![
21665                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21666                round_with_notes(2, Some("二回目の内部修正報告です。")),
21667                passing_round(3, Some(&head)),
21668            ];
21669            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21670            ok_gate(&mut f.runner.state, Some(&head));
21671            let captured = f.repo.join("public-prompt.txt");
21672            let author = &mut f.runner.state.config.agents[0];
21673            author.command = vec![
21674                "sh".into(),
21675                "-c".into(),
21676                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21677            ];
21678            author.env.insert(
21679                "PROMPT_FILE".into(),
21680                captured.to_string_lossy().into_owned(),
21681            );
21682            let m = posted(&mut f).await;
21683            assert!(m.body.contains("Adds `own.txt`"));
21684            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21685            assert!(m.body.contains("R1-1-1"));
21686            assert!(
21687                m.body
21688                    .contains("Gate (`verify.gate`): passed on the final commit")
21689            );
21690            assert!(
21691                m.body
21692                    .contains("Tests (`verify.e2e`): passed on the final commit")
21693            );
21694            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21695            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21696            let prompt = std::fs::read_to_string(captured).unwrap();
21697            assert!(prompt.contains(&format!(
21698                "git diff {} {head}",
21699                f.runner.state.change_summary.as_ref().unwrap().base
21700            )));
21701            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21702            assert_eq!(
21703                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21704                "二回目の内部修正報告です。"
21705            );
21706        }
21707    }
21708
21709    #[tokio::test]
21710    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21711        for review_only in [false, true] {
21712            let mut f = summary_fixture();
21713            if !review_only {
21714                f.runner.state.candidates[0].agent = "alpha".into();
21715            }
21716            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21717            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21718            std::fs::write(&f.reply, "no usable description").unwrap();
21719            let failed = posted(&mut f).await;
21720            assert!(
21721                failed
21722                    .body
21723                    .contains("No description of what this change does could be generated")
21724            );
21725            assert!(failed.body.is_ascii());
21726            assert!(f.runner.state.github_text.is_none());
21727            assert!(f.runner.state.change_summary.is_none());
21728            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21729            let evidence = crate::pr_outcome::summary_evidence(
21730                &f.runner.state,
21731                crate::pr_outcome::FinalHead::At(&head),
21732            );
21733            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21734            assert!(
21735                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21736                    .exists()
21737            );
21738            f.runner.state =
21739                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21740            std::fs::write(
21741                &f.reply,
21742                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21743            )
21744            .unwrap();
21745            let retried = posted(&mut f).await;
21746            assert!(retried.body.contains("Adds the committed own file."));
21747            assert_eq!(
21748                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21749                "日本語の修正報告"
21750            );
21751        }
21752    }
21753
21754    #[tokio::test]
21755    async fn same_head_verification_and_findings_invalidate_public_prose() {
21756        let mut f = summary_fixture();
21757        posted(&mut f).await;
21758        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21759        let original = f
21760            .runner
21761            .state
21762            .change_summary
21763            .as_ref()
21764            .unwrap()
21765            .evidence
21766            .clone();
21767        ok_gate(&mut f.runner.state, Some(&head));
21768        std::fs::write(
21769            &f.reply,
21770            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21771        )
21772        .unwrap();
21773        let verified = posted(&mut f).await;
21774        assert!(verified.body.contains("now verified"));
21775        assert_ne!(
21776            f.runner.state.change_summary.as_ref().unwrap().evidence,
21777            original
21778        );
21779        let mut round = round_with_notes(1, Some("内部報告"));
21780        round.reviews.push(ReviewRecord {
21781            attempts: 0,
21782            reviewer: 1,
21783            agent: "alpha".into(),
21784            summary: "内部レビュー".into(),
21785            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21786            vote: None,
21787            failed: None,
21788            duration_ms: 0,
21789        });
21790        round
21791            .fix
21792            .as_mut()
21793            .unwrap()
21794            .rejected
21795            .push(crate::verdict::Rejection {
21796                id: "R1-1-1".into(),
21797                why: "呼び出し元から到達できない".into(),
21798            });
21799        f.runner.state.reviews = vec![round];
21800        // A description omitting the required ID is unusable, even with a title.
21801        let omitted = posted(&mut f).await;
21802        assert!(omitted.body.contains("No description"));
21803        assert!(omitted.body.contains("R1-1-1"));
21804        assert!(!omitted.body.contains("到達"));
21805        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21806        let described = posted(&mut f).await;
21807        assert!(described.body.contains("unreachable from callers"));
21808        assert!(described.body.contains("R1-1-1"));
21809        assert!(!described.body.contains("到達"));
21810        let evidence = f
21811            .runner
21812            .state
21813            .change_summary
21814            .as_ref()
21815            .unwrap()
21816            .evidence
21817            .clone();
21818        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21819        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21820        assert!(posted(&mut f).await.body.contains("rationale updated"));
21821        assert_ne!(
21822            f.runner.state.change_summary.as_ref().unwrap().evidence,
21823            evidence
21824        );
21825        // Old serialized summaries have no evidence fingerprint and must regenerate.
21826        f.runner
21827            .state
21828            .change_summary
21829            .as_mut()
21830            .unwrap()
21831            .evidence
21832            .clear();
21833        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21834        assert!(posted(&mut f).await.body.contains("current specification"));
21835    }
21836
21837    #[tokio::test]
21838    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21839        let mut f = summary_fixture();
21840        let m = posted(&mut f).await;
21841        assert_eq!(m.title, "feat(own): add the own file");
21842        assert!(
21843            m.body
21844                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21845            "{}",
21846            m.body
21847        );
21848        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21849        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21850        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21851        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21852        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21853        let kept = f.runner.state.change_summary.as_ref().unwrap();
21854        assert_eq!(
21855            (kept.head.as_str(), kept.base.as_str()),
21856            (head.as_str(), base.as_str())
21857        );
21858        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21859    }
21860
21861    #[tokio::test]
21862    async fn review_only_summary_receives_the_configured_github_language() {
21863        let mut f = summary_fixture();
21864        f.runner.state.config.graph.github_language = "ja".into();
21865        let captured = f.repo.join("summary-prompt.txt");
21866        let author = &mut f.runner.state.config.agents[0];
21867        author.command = vec![
21868            "sh".into(),
21869            "-c".into(),
21870            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21871        ];
21872        author.env.insert(
21873            "PROMPT_FILE".into(),
21874            captured.to_string_lossy().into_owned(),
21875        );
21876        std::fs::write(
21877            &f.reply,
21878            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21879        )
21880        .unwrap();
21881        let m = posted(&mut f).await;
21882        let prompt = std::fs::read_to_string(captured).unwrap();
21883        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21884        assert!(!prompt.contains("in English"));
21885        assert_eq!(m.title, "feat(own): ファイルを追加");
21886        assert!(m.body.contains("own.txt を追加します。"));
21887    }
21888
21889    #[tokio::test]
21890    async fn review_fixes_come_after_the_whole_change_summary() {
21891        let mut f = summary_fixture();
21892        f.runner.state.reviews = vec![
21893            round_with_notes(1, Some("handled the PATH case")),
21894            round_with_notes(2, None),
21895        ];
21896        let m = posted(&mut f).await;
21897        let summary = m.body.find("## Summary").expect("summary");
21898        let fixes = m.body.find("## Review rounds").expect("review rounds");
21899        assert!(summary < fixes, "{}", m.body);
21900        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21901        assert!(m.body.contains("## Final verification"), "{}", m.body);
21902    }
21903
21904    #[tokio::test]
21905    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21906        let mut f = summary_fixture();
21907        let first = posted(&mut f).await;
21908        let first_head = f.runner.state.change_summary.clone().unwrap().head;
21909        // The same head reuses the record: a different reply is not read.
21910        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21911        let again = posted(&mut f).await;
21912        assert_eq!((again.title, again.body), (first.title, first.body));
21913
21914        let g = |args: &[&str]| {
21915            let out = std::process::Command::new("git")
21916                .args(args)
21917                .current_dir(&f.repo)
21918                .quiet()
21919                .output()
21920                .expect("spawn git");
21921            assert!(out.status.success(), "git {args:?}");
21922        };
21923        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21924        g(&["add", "-A"]);
21925        g(&["commit", "-q", "-m", "fix(own): handle more"]);
21926        let second = posted(&mut f).await;
21927        let kept = f.runner.state.change_summary.clone().unwrap();
21928        assert_ne!(kept.head, first_head);
21929        assert!(second.body.contains("Not read yet."), "{}", second.body);
21930        assert!(!second.body.contains("so the first summary is read"));
21931        assert_eq!(second.title, "feat(own): other");
21932    }
21933
21934    #[tokio::test]
21935    async fn an_unavailable_summary_is_stated_and_never_invented() {
21936        let mut f = summary_fixture();
21937        std::fs::write(&f.reply, "no title line here").unwrap();
21938        let m = posted(&mut f).await;
21939        assert!(
21940            m.body
21941                .contains("No description of what this change does could be generated"),
21942            "{}",
21943            m.body
21944        );
21945        assert!(m.body.contains("internal reports remain in local records"));
21946        assert!(f.runner.state.change_summary.is_none());
21947        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21948        // A preview asks nobody and claims nothing.
21949        let winner = f.runner.state.candidates[0].clone();
21950        let preview = f
21951            .runner
21952            .guarded_pr_message(&winner, None, false)
21953            .await
21954            .unwrap()
21955            .unwrap();
21956        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21957    }
21958
21959    #[test]
21960    fn pr_message_review_single_commit_uses_its_subject() {
21961        let state = review_state(&["feat(nats): per-role user"]);
21962        let m = pr_message(&state, 'A');
21963        assert_eq!(m.title, "feat(nats): per-role user");
21964        assert!(!m.body.contains("Review the work already"), "{}", m.body);
21965        assert!(m.body.contains("## Commits under review"), "{}", m.body);
21966    }
21967
21968    #[test]
21969    fn pr_message_review_multi_commit_takes_the_oldest() {
21970        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21971        let m = pr_message(&state, 'A');
21972        assert_eq!(m.title, "feat: the change");
21973        for s in ["feat: the change", "fix: typo", "fix: again"] {
21974            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21975        }
21976    }
21977
21978    #[test]
21979    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21980        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21981            let mut state = review_state(&[first, "fix: later fixup"]);
21982            state.candidates[0].branch = "機能/ログイン".to_owned();
21983            let m = pr_message(&state, 'A');
21984            assert!(
21985                m.title.starts_with("chore: land candidate A of run"),
21986                "{}",
21987                m.title
21988            );
21989        }
21990    }
21991
21992    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
21993        BranchFacts {
21994            commits: commits
21995                .iter()
21996                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
21997                .collect(),
21998            stat: stat.to_owned(),
21999        }
22000    }
22001
22002    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22003        let mut r = review_round(true, 0, 1, 1, true, true);
22004        r.round = round;
22005        r.fix = notes.map(|n| FixRecord {
22006            agent: "fixer".to_owned(),
22007            addressed: Vec::new(),
22008            rejected: Vec::new(),
22009            notes: n.to_owned(),
22010            committed: true,
22011            failed: None,
22012            duration_ms: 0,
22013            continuation: None,
22014        });
22015        r
22016    }
22017
22018    #[test]
22019    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22020        let state = review_state(&["ignored"]);
22021        let f = facts(
22022            &[
22023                (
22024                    "fix(login): resolve PATH on macOS",
22025                    "Login shells skip rc files.",
22026                ),
22027                ("fix: address review", ""),
22028            ],
22029            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22030        );
22031        let m = pr_message_with(&state, 'A', Some(&f));
22032        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22033        assert!(
22034            m.body
22035                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22036            "{}",
22037            m.body
22038        );
22039        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22040        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22041        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22042        for banned in [
22043            "Review the work already",
22044            "no task statement",
22045            "Original task",
22046        ] {
22047            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22048        }
22049        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22050    }
22051
22052    #[test]
22053    fn pr_message_review_truncates_a_huge_first_commit_body() {
22054        let state = review_state(&["ignored"]);
22055        let f = facts(
22056            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22057            "s",
22058        );
22059        let m = pr_message_with(&state, 'A', Some(&f));
22060        assert!(m.body.len() < 30_000, "{}", m.body.len());
22061        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22062        assert!(
22063            m.body.contains("- ... 1 more commit(s)"),
22064            "{}",
22065            m.body.len()
22066        );
22067        assert!(m.body.ends_with("magi:candidate-a\n"));
22068    }
22069
22070    #[test]
22071    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22072        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22073        assert_eq!(m.title, "feat: x");
22074        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22075        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22076        assert!(!m.body.contains("no task statement"), "{}", m.body);
22077    }
22078
22079    #[test]
22080    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22081        let mut state = review_state(&["日本語の件名"]);
22082        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22083        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22084        state.candidates[0].branch = "機能/ログイン".to_owned();
22085        assert!(
22086            pr_message(&state, 'A')
22087                .title
22088                .starts_with("chore: land candidate A")
22089        );
22090    }
22091
22092    #[test]
22093    fn pr_message_review_fixes_survive_a_clean_final_round() {
22094        let mut state = review_state(&["feat: x"]);
22095        state.reviews = vec![
22096            round_with_notes(1, Some("handled the PATH case")),
22097            round_with_notes(2, None),
22098        ];
22099        let body = pr_message(&state, 'A').body;
22100        assert!(
22101            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22102            "{body}"
22103        );
22104        assert!(body.contains("### Round 1"), "{body}");
22105
22106        state.reviews = vec![
22107            round_with_notes(1, Some("first fix")),
22108            round_with_notes(2, Some("")),
22109            round_with_notes(3, Some("second fix")),
22110            round_with_notes(4, None),
22111        ];
22112        let body = pr_message(&state, 'A').body;
22113        assert!(
22114            body.contains("### Round 1") && !body.contains("first fix"),
22115            "{body}"
22116        );
22117        assert!(
22118            body.contains("### Round 3") && !body.contains("second fix"),
22119            "{body}"
22120        );
22121        assert!(body.contains("### Round 2"), "{body}");
22122    }
22123
22124    const FINAL: &str = "abcdef1234567890";
22125
22126    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22127        let mut r = round_with_notes(round, None);
22128        r.verified_head = verified.map(str::to_owned);
22129        r
22130    }
22131
22132    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22133        state.gate = vec![CommandOutcome {
22134            command: "gate".to_owned(),
22135            code: Some(0),
22136            output_tail: String::new(),
22137            duration_ms: 0,
22138            resource_blocked: false,
22139        }];
22140        state.gate_ran = true;
22141        state.gate_head = head.map(str::to_owned);
22142    }
22143
22144    fn body_at(state: &RunState) -> String {
22145        pr_message_raw(
22146            state,
22147            'A',
22148            None,
22149            crate::pr_outcome::FinalHead::At(FINAL),
22150            ChangeNote::NotAsked,
22151        )
22152        .body
22153    }
22154
22155    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22156
22157    #[test]
22158    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22159        let mut state = review_state(&["feat: x"]);
22160        state.reviews = vec![
22161            round_with_notes(1, Some(SANDBOXED)),
22162            passing_round(2, Some(FINAL)),
22163        ];
22164        ok_gate(&mut state, Some(FINAL));
22165        let body = body_at(&state);
22166        assert!(!body.contains("Handled the PATH case."), "{body}");
22167        assert!(!body.contains("could not commit"), "{body}");
22168        assert!(!body.contains("could not run"), "{body}");
22169        assert!(
22170            body.contains("The fix is committed on the branch"),
22171            "{body}"
22172        );
22173        assert!(!body.contains("sandbox"), "{body}");
22174        assert!(
22175            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22176            "{body}"
22177        );
22178        assert!(
22179            body.contains("Gate (`verify.gate`): passed on the final commit"),
22180            "{body}"
22181        );
22182        // The raw report is untouched in the record.
22183        assert!(
22184            state.reviews[0]
22185                .fix
22186                .as_ref()
22187                .unwrap()
22188                .notes
22189                .contains("could not commit")
22190        );
22191    }
22192
22193    #[test]
22194    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22195        let mut state = review_state(&["feat: x"]);
22196        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22197        r1.fix.as_mut().unwrap().committed = false;
22198        let mut r2 = passing_round(2, Some(FINAL));
22199        r2.e2e[0].code = Some(1);
22200        state.reviews = vec![r1, r2];
22201        let body = body_at(&state);
22202        assert!(!body.contains("could not commit"), "{body}");
22203        assert!(!body.contains("could not run"), "{body}");
22204        assert!(body.contains("This round produced no commit"), "{body}");
22205        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22206        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22207        assert!(!body.contains("passed"), "{body}");
22208    }
22209
22210    #[test]
22211    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22212        let mut state = review_state(&["feat: x"]);
22213        state.reviews = vec![
22214            round_with_notes(1, Some(SANDBOXED)),
22215            passing_round(2, Some("0123456789abcdef")),
22216        ];
22217        ok_gate(&mut state, Some("0123456789abcdef"));
22218        let body = body_at(&state);
22219        assert!(!body.contains("HTTP tests could not run"), "{body}");
22220        // The commit claim is still settled by the structured record.
22221        assert!(!body.contains("could not commit"), "{body}");
22222        assert!(
22223            body.contains("passed on an earlier commit (`0123456`)"),
22224            "{body}"
22225        );
22226        assert!(!body.contains("passed on the final commit"), "{body}");
22227
22228        // A gate recorded before `gate_head` existed is "not recorded".
22229        ok_gate(&mut state, None);
22230        let body = body_at(&state);
22231        assert!(
22232            body.contains("passed on a commit that was not recorded"),
22233            "{body}"
22234        );
22235    }
22236
22237    #[test]
22238    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22239        let state = state_with_summary(
22240            "add retries to the client",
22241            "TITLE: feat: retries\n\nDid it.",
22242        );
22243        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22244        assert_eq!(m.title, "feat: retries");
22245        assert!(
22246            m.body.contains("<summary>Original task</summary>"),
22247            "{}",
22248            m.body
22249        );
22250        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22251        assert!(
22252            m.body
22253                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22254            "{}",
22255            m.body
22256        );
22257    }
22258
22259    #[test]
22260    fn pr_message_review_bounds_a_long_english_subject() {
22261        let long = format!("feat: {}", "word ".repeat(100));
22262        let m = pr_message(&review_state(&[&long]), 'A');
22263        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22264        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22265    }
22266
22267    #[test]
22268    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22269        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22270        let m = pr_message(&state, 'A');
22271        assert_eq!(m.title, "add retries");
22272        assert!(m.body.contains("<summary>Original task</summary>"));
22273        assert!(!m.body.contains("Commits under review"));
22274        assert_eq!(landing_subject_source(&state), state.instruction);
22275    }
22276
22277    #[test]
22278    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22279        let state = review_state(&["feat: the change", "fix: typo"]);
22280        let source = landing_subject_source(&state);
22281        assert_eq!(land::merge_subject("", &source), "feat: the change");
22282        assert_eq!(
22283            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22284            "feat: the change"
22285        );
22286        // An operator's rename still wins.
22287        assert_eq!(
22288            land::merge_subject("feat: renamed by hand", &source),
22289            "feat: renamed by hand"
22290        );
22291        let blank = review_state(&["日本語"]);
22292        assert!(
22293            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22294        );
22295    }
22296
22297    #[test]
22298    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22299        let state = review_state(&["feat: the change"]);
22300        let source = landing_subject_source(&state);
22301        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22302        assert_eq!(
22303            land::merge_subject(landing_title(&state, old), &source),
22304            "feat: the change"
22305        );
22306        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22307        let task = state_with_summary("add retries", "");
22308        assert_eq!(landing_title(&task, old), old);
22309    }
22310
22311    #[test]
22312    fn pr_message_describes_the_change_not_the_task() {
22313        let state = state_with_summary(
22314            "今回やってほしいこと: results projector を直す",
22315            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22316        );
22317        let m = pr_message(&state, 'A');
22318        assert_eq!(m.title, "fix(web): batch the runs list reads");
22319        assert!(
22320            m.body.starts_with("## Summary\n\n- reads run.json once"),
22321            "{}",
22322            m.body
22323        );
22324        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22325        let task_at = m.body.find("今回やってほしいこと").unwrap();
22326        let details_at = m.body.find("<details>").unwrap();
22327        assert!(
22328            details_at < task_at,
22329            "the task lives inside <details>: {}",
22330            m.body
22331        );
22332        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22333        assert!(m.body.contains("magi:candidate-a"));
22334    }
22335
22336    #[test]
22337    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22338        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22339        let m = pr_message(&state, 'A');
22340        assert_eq!(m.title, "add retries");
22341        assert!(
22342            m.body.contains("## Summary\n\n- did some things"),
22343            "{}",
22344            m.body
22345        );
22346
22347        let none = RunState::new(
22348            PathBuf::from("/repo"),
22349            "main".to_owned(),
22350            "abc1234".to_owned(),
22351            "add retries".to_owned(),
22352            Config::default(),
22353        );
22354        let m = pr_message(&none, 'A');
22355        assert_eq!(m.title, "add retries");
22356        assert!(!m.body.contains("## Summary"), "{}", m.body);
22357    }
22358
22359    #[test]
22360    fn pr_message_refuses_the_candidate_commit_subject() {
22361        for bad in [
22362            "TITLE: magi: candidate A (uncommitted work)",
22363            "TITLE: chore: stuff (uncommitted work)",
22364            "TITLE:   ",
22365        ] {
22366            let state = state_with_summary("add retries", bad);
22367            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22368        }
22369    }
22370
22371    #[test]
22372    fn pr_message_bounds_a_very_long_task_and_title() {
22373        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22374        let state = state_with_summary(&long, "- nothing");
22375        let m = pr_message(&state, 'A');
22376        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22377        assert!(!m.title.contains('\n'));
22378
22379        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22380        let m = pr_message(&state, 'A');
22381        assert!(m.title.starts_with("feat: "));
22382        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22383        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22384    }
22385
22386    fn long_title_of(instruction: &str) -> String {
22387        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22388    }
22389
22390    #[test]
22391    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22392        let first = "Make the landing path keep a readable title for long tasks";
22393        let line = format!(
22394            "{first}. {}",
22395            "And then keep going with more words ".repeat(20)
22396        );
22397        let t = long_title_of(&line);
22398        assert_eq!(t, first);
22399        assert!(!t.starts_with("chore: land"));
22400    }
22401
22402    #[test]
22403    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22404        let line = "word ".repeat(200);
22405        let t = long_title_of(&line);
22406        assert!(t.ends_with("word..."), "{t}");
22407        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22408    }
22409
22410    #[test]
22411    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22412        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22413            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22414        }
22415    }
22416
22417    #[test]
22418    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22419        let mut state = state_with_summary("再試行を追加する", "- nothing");
22420        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22421        state.config.graph.github_language = "ja".into();
22422        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22423    }
22424
22425    #[test]
22426    fn pr_message_title_limit_is_exact() {
22427        let at = "a".repeat(PR_TITLE_MAX);
22428        assert_eq!(long_title_of(&at), at);
22429        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22430        assert!(over.ends_with("..."), "{over}");
22431        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22432    }
22433
22434    #[test]
22435    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22436        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22437        let t = long_title_of(&line);
22438        assert!(t.ends_with("..."), "{t}");
22439        assert!(t.is_ascii(), "{t}");
22440    }
22441
22442    #[test]
22443    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22444        let line = format!(
22445            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22446            "plus more filler words ".repeat(20)
22447        );
22448        let t = long_title_of(&line);
22449        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22450    }
22451
22452    #[test]
22453    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22454        let line = format!("\n\n# {}", "title words ".repeat(40));
22455        let state = state_with_summary(&line, "- nothing");
22456        let m = pr_message(&state, 'A');
22457        assert!(m.title.starts_with("title words"), "{}", m.title);
22458        assert_eq!(
22459            land::merge_subject(&m.title, &landing_subject_source(&state)),
22460            m.title
22461        );
22462        // An operator's rename wins untouched.
22463        assert_eq!(
22464            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22465            "feat: renamed by hand"
22466        );
22467    }
22468
22469    #[tokio::test]
22470    async fn github_text_rewrite_is_bounded_and_falls_back() {
22471        crate::run::pin_test_home();
22472        for (reply, accepted) in [
22473            (
22474                "TITLE: fix: retries\nAdd retries for failed requests.",
22475                true,
22476            ),
22477            (
22478                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22479                false,
22480            ),
22481            (
22482                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22483                true,
22484            ),
22485            (
22486                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22487                true,
22488            ),
22489            (
22490                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22491                true,
22492            ),
22493            (
22494                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22495                false,
22496            ),
22497            ("TITLE: fix: retries\n", false),
22498        ] {
22499            let dir = tempfile::tempdir().unwrap();
22500            let mut runner = runner_at(RunStatus::Gating);
22501            runner.state = state_with_summary(
22502                "add retries",
22503                "TITLE: fix: retries\n日本語の説明を書きます。",
22504            );
22505            runner.state.repo = dir.path().to_owned();
22506            runner.state.candidates[0].worktree = dir.path().to_owned();
22507            let mut author = spec("alpha");
22508            author.command = vec![
22509                "sh".into(),
22510                "-c".into(),
22511                "printf '%s' \"$REWRITE_REPLY\"".into(),
22512            ];
22513            author.env.insert("REWRITE_REPLY".into(), reply.into());
22514            runner.state.config.agents = vec![author];
22515            seed_public_summary(&mut runner);
22516            let winner = runner.state.candidates[0].clone();
22517            let first = runner
22518                .guarded_pr_message(&winner, None, true)
22519                .await
22520                .unwrap();
22521            let message = if accepted {
22522                first.expect("a passing rewrite never asks")
22523            } else {
22524                // Withheld: the run parks on a question and posts nothing yet.
22525                assert!(first.is_none());
22526                assert!(runner.state.parked);
22527                let store = ask::Questions::open();
22528                let asked: Vec<_> = store
22529                    .list()
22530                    .into_iter()
22531                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22532                    .collect();
22533                assert_eq!(asked.len(), 1);
22534                assert!(!asked[0].detail.contains("secret"));
22535                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22536                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22537                assert_eq!(
22538                    asked[0]
22539                        .choices
22540                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22541                    rewrite.is_some()
22542                );
22543                // Silence: the question lapses and the neutral text is used,
22544                // without a second rewrite call or a second question.
22545                store
22546                    .update(&asked[0].id, |q| {
22547                        q.abandon("test");
22548                        Ok(())
22549                    })
22550                    .unwrap();
22551                let again = runner
22552                    .guarded_pr_message(&winner, None, true)
22553                    .await
22554                    .unwrap()
22555                    .expect("an abandoned question falls back");
22556                let n = store
22557                    .list()
22558                    .into_iter()
22559                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22560                    .count();
22561                assert_eq!(n, 1, "asked once per rejected text");
22562                again
22563            };
22564            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22565            assert!(!message.body.contains("secret"));
22566            assert!(!message.body.contains("prefix"));
22567            assert!(!message.title.contains("secret"));
22568            assert_eq!(
22569                message.body.contains("Add retries for failed requests."),
22570                accepted
22571            );
22572            assert_eq!(
22573                runner.state.seats["github-text-rewrite"].turns, 1,
22574                "only one rewrite invocation"
22575            );
22576            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22577            if accepted {
22578                assert!(runner.state.github_text.is_none());
22579                assert!(!runner.state.parked);
22580                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22581                assert!(message.body.contains("## Final verification"));
22582                assert!(message.body.contains("Evidence: final diff"));
22583                assert!(
22584                    message
22585                        .body
22586                        .contains(&format!("magi:run/{}", runner.state.id))
22587                );
22588            }
22589            if !accepted {
22590                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22591                assert!(
22592                    message
22593                        .body
22594                        .contains(&format!("magi:run/{}", runner.state.id))
22595                );
22596            }
22597        }
22598    }
22599
22600    fn scripted(id: &str, reply: &str) -> AgentSpec {
22601        let mut a = spec(id);
22602        a.command = vec![
22603            "sh".into(),
22604            "-c".into(),
22605            "printf '%s' \"$REWRITE_REPLY\"".into(),
22606        ];
22607        a.env.insert("REWRITE_REPLY".into(), reply.into());
22608        a
22609    }
22610
22611    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22612    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22613
22614    /// Gate tests inject generated public output, not internal reports.
22615    fn seed_public_summary(runner: &mut Runner) {
22616        let repo = &runner.state.repo;
22617        let g = |args: &[&str]| {
22618            let out = std::process::Command::new("git")
22619                .args(args)
22620                .current_dir(repo)
22621                .quiet()
22622                .output()
22623                .unwrap();
22624            assert!(out.status.success(), "git {args:?}");
22625            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22626        };
22627        g(&["init", "-q", "-b", "main"]);
22628        g(&[
22629            "-c",
22630            "user.name=Test",
22631            "-c",
22632            "user.email=test@example.invalid",
22633            "commit",
22634            "-q",
22635            "--allow-empty",
22636            "-m",
22637            "feat: fixture",
22638        ]);
22639        let head = g(&["rev-parse", "HEAD"]);
22640        g(&["update-ref", "refs/remotes/origin/main", &head]);
22641        let evidence = crate::pr_outcome::summary_evidence(
22642            &runner.state,
22643            crate::pr_outcome::FinalHead::At(&head),
22644        );
22645        let generated = &runner.state.candidates[0].summary;
22646        runner.state.change_summary = Some(crate::run::ChangeSummary {
22647            base: head.clone(),
22648            head,
22649            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22650            title: summary_title(generated).unwrap(),
22651            body: summary_without_title(generated),
22652        });
22653    }
22654
22655    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22656        let mut runner = runner_at(RunStatus::Gating);
22657        runner.state = state_with_summary("add retries", JA_SUMMARY);
22658        if review_only {
22659            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22660        }
22661        runner.state.repo = dir.to_owned();
22662        runner.state.candidates[0].worktree = dir.to_owned();
22663        runner.state.config.agents = agents;
22664        seed_public_summary(&mut runner);
22665        runner
22666    }
22667
22668    fn gate_questions(runner: &Runner) -> usize {
22669        ask::Questions::open()
22670            .list()
22671            .into_iter()
22672            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22673            .count()
22674    }
22675
22676    #[tokio::test]
22677    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22678        crate::run::pin_test_home();
22679        let dir = tempfile::tempdir().unwrap();
22680        let mut runner = rewrite_runner(
22681            dir.path(),
22682            true,
22683            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22684        );
22685        runner.state.config.roles.language_judge =
22686            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22687        let winner = runner.state.candidates[0].clone();
22688        let msg = runner
22689            .guarded_pr_message(&winner, None, true)
22690            .await
22691            .unwrap()
22692            .expect("an English rewrite posts without asking");
22693        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22694        assert!(msg.body.contains("<details>kept</details>"));
22695        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22696        assert_eq!(gate_questions(&runner), 0);
22697        assert!(runner.state.github_text.is_none());
22698        assert!(!runner.state.seats.contains_key("impl-A"));
22699        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22700    }
22701
22702    #[tokio::test]
22703    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22704        crate::run::pin_test_home();
22705        let dir = tempfile::tempdir().unwrap();
22706        let mut runner = rewrite_runner(
22707            dir.path(),
22708            false,
22709            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22710        );
22711        runner.state.config.roles.language_judge =
22712            Some(crate::config::AgentChoice::One("beta".into()));
22713        runner.state.seats.insert(
22714            "impl-A".into(),
22715            SeatState::new("impl-A", "alpha", runner.state.seed),
22716        );
22717        let winner = runner.state.candidates[0].clone();
22718        let msg = runner
22719            .guarded_pr_message(&winner, None, true)
22720            .await
22721            .unwrap()
22722            .expect("rewritten");
22723        assert!(msg.body.contains("<details>kept</details>"));
22724        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22725        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22726        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22727    }
22728
22729    #[tokio::test]
22730    async fn rewrite_reuses_a_resumable_implementer_session() {
22731        crate::run::pin_test_home();
22732        let dir = tempfile::tempdir().unwrap();
22733        let mut runner = rewrite_runner(
22734            dir.path(),
22735            false,
22736            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22737        );
22738        runner.state.config.roles.language_judge =
22739            Some(crate::config::AgentChoice::One("beta".into()));
22740        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22741        seat.turns = 1;
22742        runner.state.seats.insert("impl-A".into(), seat);
22743        let winner = runner.state.candidates[0].clone();
22744        runner
22745            .guarded_pr_message(&winner, None, true)
22746            .await
22747            .unwrap()
22748            .expect("rewritten");
22749        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22750        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22751    }
22752
22753    #[tokio::test]
22754    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22755        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22756        // session, was out of quota, and nothing else was asked.
22757        crate::run::pin_test_home();
22758        let dir = tempfile::tempdir().unwrap();
22759        let mut exhausted = spec("alpha");
22760        exhausted.command = vec!["false".into()];
22761        let mut runner = rewrite_runner(
22762            dir.path(),
22763            false,
22764            vec![exhausted, scripted("beta", EN_REWRITE)],
22765        );
22766        // The implementer's own agent is in the role too: it is not asked twice.
22767        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22768            "alpha".into(),
22769            "beta".into(),
22770        ]));
22771        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22772        seat.turns = 1;
22773        runner.state.seats.insert("impl-A".into(), seat);
22774        let winner = runner.state.candidates[0].clone();
22775        let msg = runner
22776            .guarded_pr_message(&winner, None, true)
22777            .await
22778            .unwrap()
22779            .expect("the role rewrites after the implementer fails");
22780        assert!(msg.body.contains("`src/net.rs`"));
22781        assert_eq!(
22782            runner.state.seats["impl-A"].turns, 2,
22783            "the implementer was tried once"
22784        );
22785        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22786        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22787    }
22788
22789    #[tokio::test]
22790    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22791        crate::run::pin_test_home();
22792        let dir = tempfile::tempdir().unwrap();
22793        let mut broken = spec("bad");
22794        broken.command = vec!["false".into()];
22795        let mut runner = rewrite_runner(
22796            dir.path(),
22797            true,
22798            vec![
22799                broken,
22800                scripted("noformat", "Add retries."),
22801                scripted("good", EN_REWRITE),
22802            ],
22803        );
22804        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22805            "bad".into(),
22806            "noformat".into(),
22807            "good".into(),
22808        ]));
22809        let winner = runner.state.candidates[0].clone();
22810        let msg = runner
22811            .guarded_pr_message(&winner, None, true)
22812            .await
22813            .unwrap()
22814            .expect("the third agent rewrites");
22815        assert!(msg.body.contains("`src/net.rs`"));
22816        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22817        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22818    }
22819
22820    #[tokio::test]
22821    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22822        crate::run::pin_test_home();
22823        // (agents, role): still Japanese; no TITLE; nothing resolvable.
22824        for (agents, role, saved) in [
22825            (
22826                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22827                Some("beta"),
22828                true,
22829            ),
22830            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22831            (Vec::new(), None, false),
22832        ] {
22833            let dir = tempfile::tempdir().unwrap();
22834            let mut runner = rewrite_runner(dir.path(), true, agents);
22835            runner.state.config.roles.language_judge =
22836                role.map(|r| crate::config::AgentChoice::One(r.into()));
22837            let winner = runner.state.candidates[0].clone();
22838            let first = runner
22839                .guarded_pr_message(&winner, None, true)
22840                .await
22841                .unwrap();
22842            assert!(first.is_none(), "withheld text parks the run");
22843            assert!(runner.state.parked);
22844            assert_eq!(gate_questions(&runner), 1);
22845            let g = runner.state.github_text.as_ref().unwrap();
22846            assert_eq!(g.rewrite.is_some(), saved);
22847            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22848        }
22849    }
22850
22851    #[tokio::test]
22852    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22853        crate::run::pin_test_home();
22854        let dir = tempfile::tempdir().unwrap();
22855        let mut runner = runner_at(RunStatus::Gating);
22856        runner.state =
22857            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22858        runner.state.repo = dir.path().to_owned();
22859        runner.state.candidates[0].worktree = dir.path().to_owned();
22860        let mut author = spec("alpha");
22861        author.command = vec![
22862            "sh".into(),
22863            "-c".into(),
22864            "printf '%s' \"$REWRITE_REPLY\"".into(),
22865        ];
22866        author.env.insert(
22867            "REWRITE_REPLY".into(),
22868            format!(
22869                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22870                "English sentence. ".repeat(600)
22871            ),
22872        );
22873        runner.state.config.agents = vec![author];
22874        seed_public_summary(&mut runner);
22875        let winner = runner.state.candidates[0].clone();
22876        assert!(
22877            runner
22878                .guarded_pr_message(&winner, None, true)
22879                .await
22880                .unwrap()
22881                .is_none()
22882        );
22883        let record = runner.state.github_text.clone().unwrap();
22884        let rewrite = record.rewrite.clone().unwrap();
22885        assert_eq!(rewrite.failed, ["body-language"]);
22886        assert!(
22887            rewrite
22888                .body
22889                .ends_with(&format!("magi:run/{}", runner.state.id))
22890        );
22891        let store = ask::Questions::open();
22892        let q = store.get(record.question.as_ref().unwrap()).unwrap();
22893        assert!(
22894            q.choices
22895                .contains(&crate::github_text::USE_REWRITE.to_owned())
22896        );
22897        for label in [
22898            "ORIGINAL",
22899            "FALLBACK",
22900            "REWRITE",
22901            "Rules still failed: body-language",
22902            "truncated",
22903        ] {
22904            assert!(q.detail.contains(label), "{label}");
22905        }
22906        let artifact = std::fs::read_to_string(crate::run::artifact_path(
22907            &runner.state,
22908            &crate::github_text::artifact_name(&record.fingerprint),
22909        ))
22910        .unwrap();
22911        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22912        // Restart from serialized state, then retry after a bad owner title.
22913        runner.state =
22914            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22915        store
22916            .update(&q.id, |q| {
22917                q.say("まだ日本語")?;
22918                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22919            })
22920            .unwrap();
22921        assert!(
22922            runner
22923                .guarded_pr_message(&winner, None, true)
22924                .await
22925                .unwrap()
22926                .is_none()
22927        );
22928        let retry = runner.state.github_text.as_ref().unwrap();
22929        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22930        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22931        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22932        assert!(
22933            q.posting_comparison
22934                .as_ref()
22935                .unwrap()
22936                .matches_detail(&q.detail)
22937        );
22938        store
22939            .update(&q.id, |q| {
22940                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22941            })
22942            .unwrap();
22943        let message = runner
22944            .guarded_pr_message(&winner, None, true)
22945            .await
22946            .unwrap()
22947            .unwrap();
22948        assert_eq!(message.title, rewrite.title);
22949        assert_eq!(message.body, rewrite.body);
22950        assert_eq!(
22951            runner.state.github_text.as_ref().unwrap().choice,
22952            Some(GithubTextChoice::Rewrite)
22953        );
22954        let again = runner
22955            .guarded_pr_message(&winner, None, true)
22956            .await
22957            .unwrap()
22958            .unwrap();
22959        assert_eq!(again.title, message.title);
22960        assert_eq!(again.body, message.body);
22961        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22962        assert!(
22963            !runner
22964                .state
22965                .events
22966                .iter()
22967                .any(|e| e.message.contains(&rewrite.title)
22968                    || e.message.contains("English sentence."))
22969        );
22970    }
22971
22972    #[tokio::test]
22973    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22974        crate::run::pin_test_home();
22975        let dir = tempfile::tempdir().unwrap();
22976        let mut runner = runner_at(RunStatus::Gating);
22977        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22978        runner.state.repo = dir.path().to_owned();
22979        runner.state.candidates[0].worktree = dir.path().to_owned();
22980        seed_public_summary(&mut runner);
22981        let winner = runner.state.candidates[0].clone();
22982        assert!(
22983            runner
22984                .guarded_pr_message(&winner, None, true)
22985                .await
22986                .unwrap()
22987                .is_none()
22988        );
22989        let record = runner.state.github_text.as_ref().unwrap();
22990        assert!(record.rewrite.is_none());
22991        // Records written before the additive field deserialize with no rewrite.
22992        let mut old = serde_json::to_value(record).unwrap();
22993        old.as_object_mut().unwrap().remove("rewrite");
22994        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
22995        assert!(old.rewrite.is_none());
22996        let id = record.question.clone().unwrap();
22997        ask::Questions::open()
22998            .update(&id, |q| {
22999                q.choices.push(crate::github_text::USE_REWRITE.into());
23000                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23001            })
23002            .unwrap();
23003        let message = runner
23004            .guarded_pr_message(&winner, None, true)
23005            .await
23006            .unwrap()
23007            .unwrap();
23008        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23009        assert_eq!(
23010            runner.state.github_text.as_ref().unwrap().choice,
23011            Some(GithubTextChoice::Fallback)
23012        );
23013    }
23014
23015    #[tokio::test]
23016    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23017        crate::run::pin_test_home();
23018        let dir = tempfile::tempdir().unwrap();
23019        let mut runner = runner_at(RunStatus::Gating);
23020        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23021        runner.state.repo = dir.path().to_owned();
23022        runner.state.candidates[0].worktree = dir.path().to_owned();
23023        seed_public_summary(&mut runner);
23024        let winner = runner.state.candidates[0].clone();
23025        assert!(
23026            runner
23027                .guarded_pr_message(&winner, None, true)
23028                .await
23029                .unwrap()
23030                .is_none()
23031        );
23032        let store = ask::Questions::open();
23033        let find = |run: &str| {
23034            store
23035                .list()
23036                .into_iter()
23037                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23038                .unwrap()
23039        };
23040        let run = runner.state.id.clone();
23041        let q = find(&run);
23042        let fp = runner
23043            .state
23044            .github_text
23045            .as_ref()
23046            .unwrap()
23047            .fingerprint
23048            .clone();
23049        let name = crate::github_text::artifact_name(&fp);
23050        let path = crate::run::artifact_path(&runner.state, &name);
23051        assert!(q.detail.contains(&path.display().to_string()));
23052        assert!(q.detail.contains("title-language: title"));
23053        let saved = std::fs::read_to_string(&path).expect("artifact written");
23054        assert!(saved.contains("日本語のタイトル"));
23055        assert!(q.cwd.is_none());
23056        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23057        // A bad replacement is refused and asked about once more.
23058        let say = |id: &str, text: &str, choice: &str| {
23059            store
23060                .update(id, |q| {
23061                    q.thread.push(ask::Turn {
23062                        who: ask::Who::Operator,
23063                        body: text.to_owned(),
23064                        at: jiff::Timestamp::now(),
23065                        note: None,
23066                    });
23067                    q.answer(ask::Answer::Choice(choice.to_owned()))
23068                })
23069                .unwrap();
23070        };
23071        say(&q.id, "まだ日本語", "use my text");
23072        assert!(
23073            runner
23074                .guarded_pr_message(&winner, None, true)
23075                .await
23076                .unwrap()
23077                .is_none()
23078        );
23079        let second = find(&run);
23080        assert_ne!(second.id, q.id);
23081        assert!(second.detail.contains("did not pass"));
23082        // The retry still compares the original text; the rejected
23083        // replacement is named by category only.
23084        assert!(second.detail.contains("日本語のタイトル"));
23085        assert!(!second.detail.contains("まだ日本語"));
23086        assert!(second.detail.contains(&path.display().to_string()));
23087        say(&second.id, "fix: retry failed requests", "use my text");
23088        let message = runner
23089            .guarded_pr_message(&winner, None, true)
23090            .await
23091            .unwrap()
23092            .expect("a vetted replacement is posted");
23093        assert_eq!(message.title, "fix: retry failed requests");
23094        assert_eq!(
23095            runner
23096                .state
23097                .github_text
23098                .as_ref()
23099                .unwrap()
23100                .chosen_title
23101                .as_deref(),
23102            Some("fix: retry failed requests")
23103        );
23104        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23105        // Resumed again: the saved decision is applied, nothing is re-asked.
23106        let again = runner
23107            .guarded_pr_message(&winner, None, true)
23108            .await
23109            .unwrap()
23110            .unwrap();
23111        assert_eq!(again.title, message.title);
23112        assert_eq!(
23113            store
23114                .list()
23115                .into_iter()
23116                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23117                .count(),
23118            2
23119        );
23120    }
23121
23122    #[test]
23123    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23124        // What magi itself writes stays English under any configured language,
23125        // so a future localisation of these headings fails here. (The agents'
23126        // own text is also checked by the posting gate.)
23127        let mut state = state_with_summary(
23128            "add retries",
23129            "TITLE: fix(web): batch reads\n- reads run.json once",
23130        );
23131        state.config.graph.language = "ja".to_owned();
23132        let m = pr_message(&state, 'A');
23133        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23134        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23135
23136        // The task is the operator's own text: it goes in untouched, and the
23137        // fallback title (no summary) may be in its language too.
23138        let task = "今回やってほしいこと: results projector を直す";
23139        let mut state = state_with_summary(task, "- no title line");
23140        state.config.graph.language = "ja".to_owned();
23141        let m = pr_message(&state, 'A');
23142        assert_eq!(
23143            m.title,
23144            format!("chore: land candidate A of run {}", state.id)
23145        );
23146        assert!(
23147            m.body.contains(&format!(
23148                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23149            )),
23150            "{}",
23151            m.body
23152        );
23153    }
23154
23155    #[test]
23156    fn pr_message_scrubs_home_paths_and_addresses() {
23157        let state = state_with_summary(
23158            "fix it in /Users/someone/src/x",
23159            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23160        );
23161        let m = pr_message(&state, 'A');
23162        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23163            assert!(!m.body.contains(leak), "{}", m.body);
23164        }
23165        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23166    }
23167
23168    #[test]
23169    fn pr_message_survives_a_task_that_closes_details() {
23170        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23171        let m = pr_message(&state, 'A');
23172        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23173    }
23174
23175    #[test]
23176    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23177        let cmd = manual_merge_command(
23178            MergeStyle::Squash,
23179            Path::new("/repo"),
23180            "b",
23181            "fix: \"quoted\" $(x) `y`\n\nbody",
23182        );
23183        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23184    }
23185
23186    #[test]
23187    fn manual_merge_command_matches_the_configured_style() {
23188        let repo = Path::new("/repo");
23189        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23190
23191        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23192        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23193
23194        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23195        assert_eq!(
23196            squash,
23197            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23198             \"Merge magi run 0832 (candidate A)\""
23199        );
23200
23201        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23202        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23203    }
23204
23205    #[test]
23206    fn a_nudge_gets_a_quarter_of_the_budget() {
23207        // The judge and implement budgets magi ships with.
23208        assert_eq!(retry_budget(secs(1200), true), secs(300));
23209        assert_eq!(retry_budget(secs(3600), true), secs(900));
23210    }
23211
23212    #[test]
23213    fn a_resent_prompt_keeps_the_whole_budget() {
23214        // The seat kept no context, so the retry is the original job again and
23215        // shortening it would only guarantee a second failure.
23216        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23217        assert_eq!(retry_budget(secs(60), false), secs(60));
23218    }
23219
23220    #[test]
23221    fn the_floor_never_exceeds_the_original_budget() {
23222        // A short configured timeout must not be *raised* by the floor: the
23223        // operator asked for a bound, and a retry may not outlast the attempt
23224        // it is retrying.
23225        assert_eq!(retry_budget(secs(60), true), secs(60));
23226        assert_eq!(retry_budget(secs(480), true), secs(120));
23227        assert_eq!(retry_budget(secs(0), true), secs(0));
23228    }
23229
23230    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23231        agent::CommandEvidence {
23232            id: "item1".to_owned(),
23233            description: "cargo test".to_owned(),
23234            exit_code,
23235            result_summary: String::new(),
23236            source: "codex".to_owned(),
23237        }
23238    }
23239
23240    #[test]
23241    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23242        // No evidence is not the same fact as unconfirmed evidence: a
23243        // backend with no adapter, or a reply that ran no commands at all,
23244        // must not be misread as carrying a dangling job.
23245        assert!(!has_unconfirmed_command(&[]));
23246    }
23247
23248    #[test]
23249    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23250        // Deliberately not a check on the exit code's *value*: a fixer
23251        // legitimately runs something that fails mid-iteration before it
23252        // succeeds, and that must never by itself reopen a valid report.
23253        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23254        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23255        assert!(!has_unconfirmed_command(&[
23256            evidence(Some(0)),
23257            evidence(Some(101))
23258        ]));
23259    }
23260
23261    #[test]
23262    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23263        assert!(has_unconfirmed_command(&[
23264            evidence(Some(0)),
23265            evidence(None)
23266        ]));
23267    }
23268
23269    #[test]
23270    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23271        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23272        assert_eq!(
23273            verified_noop_claim(true, &[], text).as_deref(),
23274            Some("already fixed by b32cfc4, on main.")
23275        );
23276    }
23277
23278    #[test]
23279    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23280        // A timeout or a bad exit code reads as the ordinary loss it is,
23281        // whatever the reply's own prose claims.
23282        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23283        assert!(verified_noop_claim(false, &[], text).is_none());
23284    }
23285
23286    #[test]
23287    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23288        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23289        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23290        // A confirmed command alongside the marker is fine.
23291        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23292    }
23293
23294    #[test]
23295    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23296        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23297    }
23298
23299    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23300    /// pair, in order, labelled A, B, C, ...
23301    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23302        runner.state.candidates = shape
23303            .iter()
23304            .enumerate()
23305            .map(|(i, &(empty, verified))| Candidate {
23306                index: i,
23307                label: (b'A' + i as u8) as char,
23308                agent: "sonnet".to_owned(),
23309                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23310                worktree: PathBuf::from(format!("/wt/{i}")),
23311                summary: String::new(),
23312                stat: String::new(),
23313                files: 0,
23314                commits: 0,
23315                empty,
23316                failed: None,
23317                verified_noop: verified.map(str::to_owned),
23318                duration_ms: 0,
23319                folded: false,
23320            })
23321            .collect();
23322    }
23323
23324    #[test]
23325    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23326        ask_test_home();
23327        let mut runner = runner_at(RunStatus::Implementing);
23328        set_candidates(
23329            &mut runner,
23330            &[
23331                (true, Some("already on main at b32cfc4")),
23332                (true, Some("same fix, see the existing test")),
23333            ],
23334        );
23335
23336        runner
23337            .after_implement()
23338            .expect("a verified no-op is not an error");
23339
23340        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23341    }
23342
23343    #[test]
23344    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23345        ask_test_home();
23346        let mut runner = runner_at(RunStatus::Implementing);
23347        // Candidate A declares a verified no-op; candidate B simply wrote
23348        // nothing and said nothing about why. One candidate's claim is not
23349        // the whole run's agreement.
23350        set_candidates(
23351            &mut runner,
23352            &[(true, Some("already on main at b32cfc4")), (true, None)],
23353        );
23354
23355        let err = runner
23356            .after_implement()
23357            .expect_err("an unverified empty candidate must still fail the run");
23358
23359        assert!(
23360            err.to_string().contains("no candidate produced a change"),
23361            "{err}"
23362        );
23363        assert_eq!(runner.state.status, RunStatus::Failed);
23364    }
23365
23366    #[test]
23367    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23368        ask_test_home();
23369        let mut runner = runner_at(RunStatus::Implementing);
23370        set_candidates(&mut runner, &[(true, None), (true, None)]);
23371
23372        let err = runner
23373            .after_implement()
23374            .expect_err("no candidate declared anything; this is an ordinary failure");
23375
23376        assert!(
23377            err.to_string().contains("no candidate produced a change"),
23378            "{err}"
23379        );
23380        assert_eq!(runner.state.status, RunStatus::Failed);
23381    }
23382}
23383
23384/// A graft rollback whose restore could not be confirmed (see
23385/// [`Runner::graft`]).
23386#[derive(Debug)]
23387struct GraftRestoreFailed(String);
23388
23389impl std::fmt::Display for GraftRestoreFailed {
23390    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23391        write!(
23392            f,
23393            "could not restore the winner to its pre-graft commit: {}",
23394            self.0
23395        )
23396    }
23397}
23398
23399impl std::error::Error for GraftRestoreFailed {}
23400
23401/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23402/// not the agent: they never draw on an agent's own timeout, and running out of
23403/// them never reads as an agent failure.
23404#[cfg(test)]
23405mod resource_wait_tests {
23406    use super::*;
23407    use std::collections::BTreeMap;
23408
23409    fn sh(id: &str, script: &str) -> AgentSpec {
23410        AgentSpec {
23411            id: id.to_owned(),
23412            kind: crate::config::AgentKind::Command,
23413            model: None,
23414            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23415            extra_args: Vec::new(),
23416            env: BTreeMap::new(),
23417            prompt_delivery: None,
23418            max_turn_secs: None,
23419        }
23420    }
23421
23422    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23423        SeatJob {
23424            seat: SeatState::new("impl-A", &spec.id, 1),
23425            spec,
23426            cwd: dir.to_path_buf(),
23427            prompt: "do it".to_owned(),
23428            timeout,
23429            allow_write: true,
23430            sessions: false,
23431            artifacts: dir.join("art"),
23432            stem: "impl-A".to_owned(),
23433            handover: None,
23434        }
23435    }
23436
23437    fn state(resource_wait_secs: u64) -> RunState {
23438        let mut config = Config::default();
23439        config.graph.timeout_resource_wait = resource_wait_secs;
23440        RunState::new(
23441            PathBuf::from("/repo"),
23442            "main".to_owned(),
23443            "abc1234".to_owned(),
23444            "task".to_owned(),
23445            config,
23446        )
23447    }
23448
23449    fn hold(cache: &Path) -> crate::cache::Guard {
23450        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23451        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23452            crate::cache::AcquireOutcome::Acquired(g) => g,
23453            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23454        }
23455    }
23456
23457    async fn run(
23458        jobs: Vec<SeatJob>,
23459        sem: Arc<Semaphore>,
23460        cache: Option<&Path>,
23461        st: &mut RunState,
23462    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23463        let prompts = Prompts::default();
23464        let ctx = WaveCtx {
23465            run: "r",
23466            node: "implement",
23467            prompts: &prompts,
23468            cache,
23469            round: None,
23470            carry_seats: false,
23471            pauses: None,
23472        };
23473        wave(jobs, sem, &ctx, st, 0).await
23474    }
23475
23476    #[tokio::test]
23477    async fn verification_cache_wait_preserves_the_command_budget() {
23478        crate::run::pin_test_home();
23479        let tmp = tempfile::tempdir().unwrap();
23480        let cache = tmp.path().join("target");
23481        let guard = hold(&cache);
23482        let release = tokio::spawn(async move {
23483            tokio::time::sleep(Duration::from_millis(100)).await;
23484            guard.release();
23485        });
23486        let mut st = state(10);
23487        let budget = Duration::from_millis(50);
23488        let wait = repository_wait(&st, None);
23489        let (outcomes, _) = with_cache_lease(
23490            &mut st,
23491            Some(&cache),
23492            "gate",
23493            "gate",
23494            tmp.path(),
23495            "abc1234",
23496            budget,
23497            "test gate",
23498            &wait,
23499            |_, available| async move {
23500                assert_eq!(
23501                    available, budget,
23502                    "contention must not shorten verification"
23503                );
23504                (Vec::new(), false, Vec::new())
23505            },
23506        )
23507        .await
23508        .unwrap();
23509        release.await.unwrap();
23510        assert!(outcomes.is_empty(), "{outcomes:?}");
23511        assert!(
23512            st.events
23513                .iter()
23514                .any(|event| event.message.contains("waiting for"))
23515        );
23516        assert!(st.resource_wait.is_none());
23517        // The verification body no longer owns the lease after returning.
23518        hold(&cache).release();
23519    }
23520
23521    #[tokio::test]
23522    async fn verification_cache_wait_expires_without_starting_commands() {
23523        crate::run::pin_test_home();
23524        let tmp = tempfile::tempdir().unwrap();
23525        let cache = tmp.path().join("target");
23526        let _held = hold(&cache);
23527        let mut st = state(0);
23528        let wait = repository_wait(&st, None);
23529        let (outcomes, _) = with_cache_lease(
23530            &mut st,
23531            Some(&cache),
23532            "gate",
23533            "gate",
23534            tmp.path(),
23535            "abc1234",
23536            Duration::from_secs(60),
23537            "test gate",
23538            &wait,
23539            |_, _| async { panic!("commands must not start without the cache lease") },
23540        )
23541        .await
23542        .unwrap();
23543        assert_eq!(outcomes.len(), 1);
23544        assert!(outcomes[0].resource_blocked);
23545        assert!(st.resource_wait.is_none());
23546        assert!(matches!(
23547            st.resource_block.unwrap().cause,
23548            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23549        ));
23550    }
23551
23552    #[tokio::test]
23553    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23554        crate::run::pin_test_home();
23555        let tmp = tempfile::tempdir().unwrap();
23556        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23557        let cache = tmp.path().join("cache");
23558        let mut state = state(21600);
23559        let wait = repository_wait(&state, None);
23560        let (outcomes, _) = with_cache_lease(
23561            &mut state,
23562            Some(&cache),
23563            "gate",
23564            "gate",
23565            tmp.path(),
23566            "head",
23567            Duration::from_secs(60),
23568            "gate",
23569            &wait,
23570            |_, _| async { panic!("freshness failure must not start verification") },
23571        )
23572        .await
23573        .unwrap();
23574        assert_eq!(outcomes.len(), 1);
23575        assert!(outcomes[0].resource_blocked);
23576        let block = state.resource_block.unwrap();
23577        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23578        assert!(!block.description().contains("timed out"));
23579    }
23580
23581    #[tokio::test]
23582    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23583        crate::run::pin_test_home();
23584        let tmp = tempfile::tempdir().unwrap();
23585        let cache = tmp.path().join("cache");
23586        let _held = hold(&cache);
23587        let mut state = state(21600);
23588        let pause = Pause::new();
23589        let interrupt = Pause::new();
23590        let cause = ParkCause::Operator;
23591        pause.park_for(cause.clone());
23592        let prompts = Prompts::default();
23593        let ctx = WaveCtx {
23594            run: "test",
23595            node: "fix",
23596            prompts: &prompts,
23597            cache: Some(&cache),
23598            round: None,
23599            carry_seats: false,
23600            pauses: Some((&pause, &interrupt)),
23601        };
23602        let result = wave(
23603            vec![job(
23604                sh("writer", "exit 99"),
23605                tmp.path(),
23606                Duration::from_secs(1),
23607            )],
23608            Arc::new(Semaphore::new(1)),
23609            &ctx,
23610            &mut state,
23611            0,
23612        )
23613        .await;
23614        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23615        assert_eq!(result[0].1.turns, 0);
23616        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23617        assert_eq!(
23618            state.resource_block.unwrap().cause,
23619            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23620        );
23621        assert!(state.active.is_empty());
23622    }
23623
23624    #[tokio::test]
23625    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23626        crate::run::pin_test_home();
23627        let tmp = tempfile::tempdir().unwrap();
23628        let cache = tmp.path().join("target-a");
23629        let guard = hold(&cache);
23630        // Held for longer than the agent's whole 1s budget.
23631        tokio::spawn(async move {
23632            tokio::time::sleep(Duration::from_millis(1500)).await;
23633            guard.release();
23634        });
23635        let mut st = state(60);
23636        let results = run(
23637            vec![job(
23638                sh("alpha", "echo ok"),
23639                tmp.path(),
23640                Duration::from_secs(1),
23641            )],
23642            Arc::new(Semaphore::new(1)),
23643            Some(&cache),
23644            &mut st,
23645        )
23646        .await;
23647        assert!(
23648            matches!(results[0].2, AgentOutcome::Ok(_)),
23649            "the wait must not be charged to the agent"
23650        );
23651        assert!(st.resource_block.is_none());
23652        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23653        assert!(st.resource_wait.is_none() && st.active.is_empty());
23654    }
23655
23656    #[tokio::test]
23657    #[cfg(unix)]
23658    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23659        use std::os::fd::AsRawFd as _;
23660        crate::run::pin_test_home();
23661        let tmp = tempfile::tempdir().unwrap();
23662        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23663        let common = tmp.path().join(".git");
23664        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23665        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23666        assert_eq!(
23667            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23668            0
23669        );
23670        let mut st = state(1);
23671        st.config.coordination = crate::config::test_coordination();
23672        let mut reader = job(
23673            sh("reader", "echo must-not-run"),
23674            tmp.path(),
23675            Duration::from_secs(30),
23676        );
23677        reader.allow_write = false;
23678        let results = run(
23679            vec![reader.clone()],
23680            Arc::new(Semaphore::new(1)),
23681            None,
23682            &mut st,
23683        )
23684        .await;
23685        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23686        assert_eq!(results[0].1.turns, 0);
23687        assert!(FailClass::of(&results[0].2).is_none());
23688        assert_eq!(
23689            st.resource_block.as_ref().unwrap().kind,
23690            "repository coordination"
23691        );
23692        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23693        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23694        drop(external);
23695        let editor = crate::repo_lock::SeatLease::for_checkout(
23696            tmp.path(),
23697            true,
23698            crate::config::test_coordination().as_ref(),
23699        )
23700        .await
23701        .unwrap();
23702        let pause = Pause::new();
23703        let interrupt = Pause::new();
23704        let prompts = Prompts::default();
23705        let ctx = WaveCtx {
23706            run: "r",
23707            node: "review",
23708            prompts: &prompts,
23709            cache: None,
23710            round: None,
23711            carry_seats: false,
23712            pauses: Some((&pause, &interrupt)),
23713        };
23714        let (results, ()) = tokio::join!(
23715            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23716            async {
23717                tokio::time::sleep(Duration::from_millis(50)).await;
23718                pause.park();
23719            }
23720        );
23721        assert!(
23722            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23723        );
23724        assert_eq!(results[0].1.turns, 0);
23725        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23726        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23727        drop(editor);
23728    }
23729
23730    #[tokio::test]
23731    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23732        crate::run::pin_test_home();
23733        let tmp = tempfile::tempdir().unwrap();
23734        let cache = tmp.path().join("target-b");
23735        let _held = hold(&cache);
23736        let mut st = state(1);
23737        let results = run(
23738            vec![job(
23739                sh("alpha", "echo ok"),
23740                tmp.path(),
23741                Duration::from_secs(1),
23742            )],
23743            Arc::new(Semaphore::new(1)),
23744            Some(&cache),
23745            &mut st,
23746        )
23747        .await;
23748        let (_, seat, out) = &results[0];
23749        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23750        assert!(wave_blocked(&results));
23751        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23752        assert_eq!(seat.turns, 0, "nothing was asked");
23753        assert!(
23754            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23755            "no agent was spawned"
23756        );
23757        assert!(FailClass::of(out).is_none(), "not a failure class");
23758        let block = st.resource_block.as_ref().expect("recorded for the resume");
23759        assert_eq!(block.kind, "shared build cache");
23760        assert!(matches!(
23761            block.cause,
23762            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23763        ));
23764        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23765        assert!(st.active.is_empty());
23766    }
23767
23768    #[tokio::test]
23769    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23770        crate::run::pin_test_home();
23771        let tmp = tempfile::tempdir().unwrap();
23772        let mut st = state(1);
23773        let results = run(
23774            vec![job(
23775                sh("alpha", "echo ok"),
23776                tmp.path(),
23777                Duration::from_secs(1),
23778            )],
23779            Arc::new(Semaphore::new(0)),
23780            None,
23781            &mut st,
23782        )
23783        .await;
23784        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23785        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23786        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23787    }
23788
23789    #[tokio::test]
23790    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23791        crate::run::pin_test_home();
23792        let tmp = tempfile::tempdir().unwrap();
23793        let mut st = state(60);
23794        let results = run(
23795            vec![job(
23796                sh("alpha", "sleep 30"),
23797                tmp.path(),
23798                Duration::from_secs(1),
23799            )],
23800            Arc::new(Semaphore::new(1)),
23801            None,
23802            &mut st,
23803        )
23804        .await;
23805        let out = &results[0].2;
23806        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23807        let class = FailClass::of(out).expect("a failure");
23808        assert_eq!(class, FailClass::Timeout);
23809        assert!(should_hand_over(None, &class), "existing handover policy");
23810        assert!(st.resource_block.is_none());
23811    }
23812}