Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim on one run's repair or `magi fix`, released on drop — including
534/// on an early return or a panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Recorded before the first save, so a crash right after minting
773        // cannot leave a run with no origin. Legibility only: nothing reads it
774        // to decide anything.
775        state.fetched_base = Some(fetched_base);
776        state.origin = Some(origin);
777        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
778        state.status = RunStatus::Prep;
779        state.event("start", format!("run {} created", state.id));
780        startup.finish(&mut state, Ok(()))?;
781        Ok(Self {
782            state,
783            roles,
784            sem: Arc::new(Semaphore::new(max_parallel)),
785            pause: Pause::new(),
786            interrupt: Pause::new(),
787        })
788    }
789
790    /// Open a review-only run against work that already exists on `branch`.
791    ///
792    /// The expensive half of the graph is the implement wave — measured at
793    /// 111 and 134 internal tool-loop turns on this repository, against a
794    /// handful for a judge or a reviewer. The cheap half is worth running on
795    /// hand-written work too, and there was no way to reach it.
796    ///
797    /// No new state and no schema change are needed: a run with **one** viable
798    /// candidate and a tally already decided degrades `execute` to exactly
799    /// review → gate → merge, because `judge` skips a single-candidate field,
800    /// `deliberate` has fewer than two first choices to reconcile, `vote`
801    /// returns early, `tally` is already present and `fold_losers` has no
802    /// losers. Resuming such a run therefore does the right thing as well.
803    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
804        Self::review_taking_over(repo, branch, config, None, origin).await
805    }
806
807    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
808    /// at the same task still has `branch` checked out, its worktree is
809    /// released first if that is safe (see [`crate::handover`]), and the
810    /// review refuses with the reason if it is not. `None` is a hand-run
811    /// review: it has no earlier attempts, so only a worktree of a dead run
812    /// magi recorded itself can be released.
813    pub async fn review_taking_over(
814        repo: &Path,
815        branch: &str,
816        config: Config,
817        takeover: Option<crate::handover::Takeover>,
818        origin: Origin,
819    ) -> Result<Self> {
820        let repo = git::toplevel(repo).await?;
821        let missing = agent::missing_programs(&config.agents);
822        if !missing.is_empty() {
823            bail!(
824                "these agent programs are not on PATH: {}. Fix the roster in \
825                 magi.toml or install them.",
826                missing.join(", ")
827            );
828        }
829        let base_branch =
830            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
831                .await?;
832        if base_branch == branch {
833            bail!("`{branch}` is the base branch; there is nothing to review against");
834        }
835        let fetched_base = resolve_base(
836            &repo,
837            &base_branch,
838            &config.merge.remote,
839            config.coordination.as_ref(),
840        )
841        .await?;
842        let base_commit = fetched_base.commit.clone();
843
844        let roles = config.resolve_roles()?;
845        let max_parallel = config.graph.max_parallel.max(1);
846        let mut state = RunState::new(
847            repo.clone(),
848            base_branch,
849            base_commit.clone(),
850            format!("Review the work already on branch `{branch}`."),
851            config,
852        );
853        state.fetched_base = Some(fetched_base);
854        state.origin = Some(origin);
855        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
856        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
857        let opened = async {
858            startup.stage(&mut state, "handover")?;
859
860            // Released before anything else touches the branch: a stale local
861            // branch is moved with `git branch -f`, which git refuses while an
862            // earlier attempt's worktree still has it checked out. Everything
863            // after this point that can fail puts the old run back.
864            // A hand-run review has no task, hence no earlier attempts, but a
865            // worktree of a dead run magi made may still be released.
866            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
867                earlier: Vec::new(),
868                home: crate::run::home(),
869                choice: None,
870                task_instruction: None,
871            });
872            let wait = repository_wait(&state, None);
873            let coordination_config = state.config.coordination.clone();
874            let coordination =
875                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
876                    .await?;
877            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
878            if let Some(released) = &released {
879                state.event(
880                    "release",
881                    format!(
882                        "took `{branch}` over from run {}: its worktree was released: {}",
883                        crate::run::short_of(&released.old_id),
884                        released.audit
885                    ),
886                );
887            }
888            let opened = async {
889                startup.stage(&mut state, "review-preflight")?;
890                // The owner's answer to an earlier divergence question is applied
891                // here: after the release (git will not move a checked-out branch)
892                // and before the sync that would otherwise ask again.
893                state.foreign_checkout = crate::handover::preflight_review(
894                    &repo,
895                    branch,
896                    &takeover.home,
897                    &state.config.merge.remote,
898                )
899                .await?;
900                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
901                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
902                }
903                if let Some(choice) = takeover.choice.as_ref()
904                    && let Err(e) =
905                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
906                            .await
907                {
908                    return Err(e.context("applying the owner's answer about the diverged branch"));
909                }
910                // An earlier attempt of this task that published the task head, and
911                // whose work is the branch under review, hands that head over: the
912                // review run updates the same pull request. Without one (a legacy
913                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
914                // own head and its own open pull request is adopted as before.
915                for id in takeover.earlier.iter().rev() {
916                    if let Ok(old) = RunState::load_under(id, &takeover.home)
917                        && let Some(head) = old.pr_head.clone()
918                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
919                    {
920                        state.pr_head = Some(head);
921                        break;
922                    }
923                }
924                drop(coordination);
925                startup.stage(&mut state, "open-review")?;
926                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
927            }
928            .await;
929            if opened.is_err()
930                && let Some(released) = &released
931            {
932                // Restoration is ownership checked and best effort. Keep the original
933                // startup cause even if reacquiring coordination is no longer possible.
934                match crate::repo_lock::SeatLease::operation_with(
935                    &repo,
936                    coordination_config.as_ref(),
937                    &wait,
938                )
939                .await
940                {
941                    Ok(_coordination) => {
942                        released.restore(&repo, branch).await;
943                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
944                            .map(|old| {
945                                format!(
946                                    "predecessor released_to={:?}; candidate path exists={}",
947                                    old.released_to,
948                                    old.candidates.iter().any(|c| {
949                                        c.branch == branch && c.worktree.exists()
950                                    }),
951                                )
952                            })
953                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
954                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
955                    }
956                    Err(error) => {
957                        tracing::warn!(
958                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
959                            state.id
960                        );
961                        state.event(
962                            "startup-restore",
963                            format!("restoration not attempted: {error:#}"),
964                        );
965                    }
966                }
967            }
968            opened
969        }
970        .await;
971        startup.finish(&mut state, opened)?;
972        Ok(Self {
973            state,
974            roles,
975            sem: Arc::new(Semaphore::new(max_parallel)),
976            pause: Pause::new(),
977            interrupt: Pause::new(),
978        })
979    }
980
981    /// Recover exactly the operator-selected public implementation, without touching old sources.
982    pub async fn recover_pr(
983        repo: &Path,
984        task: &str,
985        selection: crate::pr_recovery::Selection,
986        mut config: Config,
987    ) -> Result<Self> {
988        selection.configure(&mut config)?;
989        selection.check_task(task, &selection.instruction)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.origin = Some(crate::run::Origin::queue(task));
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.runs.last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.ensure_base_episode();
1417        state.record_base_sync_stop(&head);
1418        state.save()?;
1419        let fetched =
1420            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421        anyhow::ensure!(
1422            fetched.ok(),
1423            "cannot authorize base recovery without a fetched base"
1424        );
1425        let approved_tip = git::rev_parse(
1426            &state.repo,
1427            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428        )
1429        .await?;
1430        if resumption {
1431            anyhow::ensure!(
1432                crate::daemon::compatible_run_instruction(&state, &queued),
1433                "resumption instruction does not match the retained run"
1434            );
1435            let approval = queued
1436                .resumption
1437                .as_ref()
1438                .context("no operator resumption")?;
1439            anyhow::ensure!(
1440                approval.instruction == queued.instruction,
1441                "resumption approval does not match the current task instruction"
1442            );
1443            state.authorize_base_sync_resumption(task, approval)?;
1444        } else {
1445            state.authorize_base_sync(task)?;
1446        }
1447        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448        state
1449            .base_sync_recovery
1450            .authorizations
1451            .last_mut()
1452            .unwrap()
1453            .scope = Some(crate::base_sync_recovery::Scope {
1454            tip: approved_tip,
1455            instruction: queued.instruction.clone(),
1456            episode,
1457        });
1458        state.event(
1459            "base-recovery",
1460            "owner authorized a bounded base synchronization interval; fresh verification required",
1461        );
1462        state.save()?;
1463        Ok(())
1464    }
1465
1466    /// Reopen an existing run.
1467    pub fn resume(id: &str) -> Result<Self> {
1468        let state = RunState::load(id)?;
1469        if state.startup.as_ref().is_some_and(|s| !s.complete)
1470            || (state.status == RunStatus::Prep
1471                && state.instruction.trim().is_empty()
1472                && state.candidates.is_empty()
1473                && state.prep_complete == Some(false))
1474        {
1475            bail!(
1476                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1477            );
1478        }
1479        crate::pr_head::ensure_owner(&state)?;
1480        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1481            bail!(
1482                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1483            );
1484        }
1485        if let Some(to) = &state.released_to {
1486            bail!(
1487                "run {} cannot be resumed: its worktree was released to run {}",
1488                state.short(),
1489                crate::run::short_of(to)
1490            );
1491        }
1492        let roles = state.config.resolve_roles()?;
1493        if let Some(id) = &state.config.graph.selected_implementer
1494            && state.candidates.iter().any(|c| c.agent != *id)
1495        {
1496            bail!(
1497                "replacement implementer pin does not match the saved candidates; task must stay held"
1498            );
1499        }
1500        let max_parallel = state.config.graph.max_parallel.max(1);
1501        Ok(Self {
1502            state,
1503            roles,
1504            sem: Arc::new(Semaphore::new(max_parallel)),
1505            pause: Pause::new(),
1506            interrupt: Pause::new(),
1507        })
1508    }
1509
1510    /// Walk the graph to a terminal state, skipping nodes already recorded.
1511    ///
1512    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1513    /// the phone - ends here, so this is the one place a run that ended
1514    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1515    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1516    pub async fn execute(&mut self) -> Result<()> {
1517        anyhow::ensure!(
1518            self.state.startup.as_ref().is_none_or(|s| s.complete),
1519            "startup incomplete; inspect retained evidence before explicit recovery"
1520        );
1521        if let Some(selection) = &self.state.pr_recovery
1522            && (selection.source_proof.is_some() || selection.source_run.is_some())
1523        {
1524            Box::pin(selection.verify_source(&self.state.repo)).await?;
1525        }
1526        // Queue claims cover daemon work; this also serializes manual drivers
1527        // consuming a repair answer, including the new review and decision.
1528        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1529            || self.state.land_approval.is_some()
1530            || self.state.merge_decision.is_some()
1531            || !self.state.merge_repairs.is_empty()
1532            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1533        {
1534            let claim = FixClaim::acquire(&self.state.dir())?;
1535            let disk = RunState::load(&self.state.id)?;
1536            let checkpoint = |s: &RunState| {
1537                (
1538                    s.merge_repairs.len(),
1539                    s.merge_repairs
1540                        .last()
1541                        .map(|r| (r.question.clone(), r.phase)),
1542                    s.land_approval
1543                        .as_ref()
1544                        .map(|a| (a.question.clone(), a.head.clone())),
1545                )
1546            };
1547            if checkpoint(&disk) != checkpoint(&self.state) {
1548                bail!(
1549                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1550                );
1551            }
1552            Some(claim)
1553        } else {
1554            None
1555        };
1556        let mut execution = self.claim_driver().await?;
1557        let runner = &mut *execution.runner;
1558        let wait = runner.repository_wait();
1559        // Keep the graph's large future off the callers' stack, including
1560        // standalone task attempts running on a default-sized test thread.
1561        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1562        let result = match result {
1563            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1564                record_repository_block(&mut runner.state, &e);
1565                runner.park_here()?;
1566                Ok(())
1567            }
1568            other => other,
1569        };
1570        execution.driver.finish(&mut runner.state).await?;
1571        let ended = if result.is_err() {
1572            Some({
1573                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1574                if let Err(error) = &result {
1575                    notice.message = format!(
1576                        "Run {} stopped with an error: {}",
1577                        runner.state.id,
1578                        error.to_string().lines().next().unwrap_or("unknown error")
1579                    );
1580                }
1581                notice
1582            })
1583        } else {
1584            crate::notices::run_ended(&runner.state)
1585        };
1586        if let Some(notice) = ended {
1587            crate::notices::raise(notice);
1588        }
1589        result
1590    }
1591
1592    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1593        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1594    }
1595
1596    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1597        let wait = self.repository_wait();
1598        coordinate_operation(&mut self.state, cwd, &wait).await
1599    }
1600
1601    /// Validate ownership and persist the driver claim in one critical section.
1602    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1603        let owner = match crate::pr_head::owner_lock_wait(
1604            &crate::run::home(),
1605            &self.state.id,
1606            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1607        )
1608        .await
1609        {
1610            Ok(owner) => owner,
1611            Err(error) => {
1612                let detail = format!("worktree maintenance or ownership update: {error:#}");
1613                // Admission did not own run.json: retain this resource outcome
1614                // in memory for queue settlement without overwriting a concurrent
1615                // retirement or publication transaction.
1616                self.state.resource_block = Some(crate::run::ResourceBlock {
1617                    cause: crate::run::ResourceBlockCause::Other,
1618                    node: "start".into(),
1619                    kind: "worktree maintenance".into(),
1620                    detail: detail.clone(),
1621                    at: Timestamp::now(),
1622                });
1623                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1624            }
1625        };
1626        let disk = RunState::load(&self.state.id)?;
1627        crate::pr_head::ensure_owner(&disk)?;
1628        let claimed =
1629            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1630        // Only proven-live ownership refuses here. Unknown records (legacy runs
1631        // with no claim marker or process identity) fall through to the
1632        // exclusive execution lease taken just below, which is what actually
1633        // keeps a second driver out.
1634        anyhow::ensure!(
1635            disk.liveness(claimed) != crate::run::Liveness::Live,
1636            "run already has a live driver; wait for its execution to end"
1637        );
1638        if let Some(to) = &disk.released_to {
1639            bail!(
1640                "run {} cannot continue: its worktree was released to run {}",
1641                self.state.short(),
1642                crate::run::short_of(to)
1643            );
1644        }
1645        anyhow::ensure!(
1646            serde_json::to_value(&disk.base_sync_recovery)?
1647                == serde_json::to_value(&self.state.base_sync_recovery)?,
1648            "base recovery changed since this run was loaded; reload the saved run"
1649        );
1650        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1651        let mut execution = DriverExecution {
1652            runner: self,
1653            driver,
1654        };
1655        // Declare after the execution guard: on any claim error this short
1656        // lock must drop before that guard reloads and releases ownership.
1657        let _owner = owner;
1658        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1659            .await?;
1660        let state = &mut execution.runner.state;
1661        // Moving again, so it is no longer parked. Set before the walk rather
1662        // than in `resume`, so every way of re-entering the graph clears it
1663        // and a card cannot claim a run is waiting to be resumed while the
1664        // agents are already working.
1665        state.parked = false;
1666        state.resource_wait = None;
1667        state.resource_block = None;
1668        // Any seat this state still lists as answering belongs to whatever
1669        // process last drove this run — this one included, if it crashed
1670        // mid-wave. Cleared and flushed immediately, before anything else
1671        // runs, so a resume can never show a seat as live when nothing is
1672        // asking it anything yet; the node that actually dispatches the next
1673        // wave repopulates it.
1674        state.clear_active();
1675        // Recorded in the same spot, and flushed together with the clear
1676        // above: this is the pid a reader checks (`RunState::liveness`) when
1677        // no daemon claim exists to answer "is a process still driving this
1678        // run" — a plain `magi run` / `magi review` typed into a terminal
1679        // claims nothing there. Always overwritten, never only-if-absent, so
1680        // a resumed run's stale pid from a previous, possibly-dead process
1681        // can never survive into this one's own report. Unlike
1682        // `clear_active`, this changes on every single `execute()` call, so
1683        // the save below is now unconditional rather than only-if-cleared.
1684        //
1685        // `driver_started_at` is recorded in the same breath, from this same
1686        // pid, so `liveness` can tell a live pid that is genuinely still us
1687        // apart from one the OS has since handed to an unrelated process —
1688        // see that field's own doc for why the pid alone is not enough.
1689        let pid = std::process::id();
1690        state.driver_pid = Some(pid);
1691        state.driver_started_at = crate::proc::process_started_at(pid);
1692        state.driver_exited = false;
1693        state.driver_claimed = Some(true);
1694        state.begin_pass();
1695        execution.driver.bind(state);
1696        state.save()?;
1697        execution.driver.record(state)?;
1698        Ok(execution)
1699    }
1700
1701    /// No conflict checkpoint proves dispatch was never reached. End that
1702    /// consumed generation so an owner can explicitly authorize a fresh one.
1703    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1704        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1705        if let Err(error) = &result
1706            && !error.is::<crate::repo_lock::ResourceBlocked>()
1707            && self
1708                .state
1709                .repair_grants
1710                .last()
1711                .is_some_and(|g| g.conflict.is_none())
1712        {
1713            self.state.repair_grants.last_mut().unwrap().phase =
1714                crate::repair_budget::Phase::Complete;
1715            self.state.status = RunStatus::Blocked;
1716            self.state.save()?;
1717            bail!(
1718                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1719                self.state.passes.last().map_or(0, |p| p.index)
1720            );
1721        }
1722        result
1723    }
1724
1725    /// A grant owns only these repair attempts. Confirmation reviews cannot
1726    /// create another fixer allowance, and every operation is restart-checkpointed.
1727    async fn execute_granted_repair(&mut self) -> Result<()> {
1728        use crate::repair_budget::{Kind, Phase};
1729        let index = self.state.repair_grants.len() - 1;
1730        let winner = self
1731            .state
1732            .winner()
1733            .cloned()
1734            .context("repair winner missing")?;
1735        loop {
1736            let grant = self.state.repair_grants[index].clone();
1737            match grant.phase {
1738                Phase::Pending => {
1739                    if grant.spent >= grant.receipt.request.rounds {
1740                        self.state.repair_grants[index].phase = Phase::Complete;
1741                        self.state.status = RunStatus::Blocked;
1742                        self.state.save()?;
1743                        return Ok(());
1744                    }
1745                    // Reserve before dispatch. A crash can consume an attempt,
1746                    // but can never duplicate it or reset saved fixer history.
1747                    self.state.repair_grants[index].spent += 1;
1748                    self.state.repair_grants[index].phase = Phase::Fixing;
1749                    self.state.save()?;
1750                    match grant.receipt.request.kind {
1751                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1752                        Kind::Gate => {
1753                            let failed = self.state.gate.clone();
1754                            let _ = self.gate_fix_round(&winner, &failed).await?;
1755                        }
1756                        Kind::Review => {
1757                            let mut record = self
1758                                .state
1759                                .reviews
1760                                .last()
1761                                .cloned()
1762                                .context("review failure missing")?;
1763                            let round = record.round;
1764                            let step = self.run_fix_round(&winner, &mut record).await?;
1765                            self.state.reviews[round - 1] = record;
1766                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1767                                self.state.repair_grants[index].phase = Phase::Complete;
1768                                self.state.status = RunStatus::Blocked;
1769                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1770                                self.state.save()?;
1771                                return Ok(());
1772                            }
1773                            if matches!(step, FixStep::Resource) {
1774                                self.stop_for_resource()?;
1775                                return Ok(());
1776                            }
1777                            self.state.save()?;
1778                        }
1779                    }
1780                    if self.resource_halt()? {
1781                        return Ok(());
1782                    }
1783                }
1784                Phase::Fixing => {
1785                    if grant.receipt.request.kind == Kind::BaseConflict {
1786                        self.repair_base_conflict(false).await?;
1787                    }
1788                    // Interrupted fixers may have left source changes or an
1789                    // unpushed commit. Preserve them before fresh assessment.
1790                    let _coordination = self.coordinate(&winner.worktree).await?;
1791                    let rescued = git::rescue_commit(
1792                        &winner.worktree,
1793                        "magi: preserve interrupted granted repair",
1794                    )
1795                    .await?;
1796                    self.state.note_withheld("repair-grant", &rescued.withheld);
1797                    // Fresh checks must supersede even a same-head failure.
1798                    // Archive rather than erase failure evidence or counters.
1799                    self.state
1800                        .verification_history
1801                        .push(crate::run::VerificationEvidence {
1802                            superseded_by: None,
1803                            archived_at: Some(Timestamp::now()),
1804                            fixes_spent: Some(self.state.gate_fixes.len()),
1805                            gate_head: self.state.gate_head.take(),
1806                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1807                            gate: std::mem::take(&mut self.state.gate),
1808                            e2e: self.state.head_e2e.take(),
1809                        });
1810                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1811                        self.state
1812                            .verification_history
1813                            .push(crate::run::VerificationEvidence {
1814                                superseded_by: None,
1815                                archived_at: Some(Timestamp::now()),
1816                                fixes_spent: Some(self.state.gate_fixes.len()),
1817                                gate_head: None,
1818                                gate_ran: false,
1819                                gate: Vec::new(),
1820                                e2e: Some(e2e),
1821                            });
1822                    }
1823                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1824                    self.state.repair_grants[index].head =
1825                        git::rev_parse(&winner.worktree, "HEAD").await?;
1826                    self.state.repair_grants[index].phase = Phase::Reviewing;
1827                    if grant.receipt.request.kind == Kind::BaseConflict {
1828                        self.state.base_sync_recovery.stop = None;
1829                        if let Some(sync) = &mut self.state.base_sync {
1830                            sync.tip = grant
1831                                .receipt
1832                                .request
1833                                .tip
1834                                .clone()
1835                                .context("repair base tip missing")?;
1836                            sync.conflict = None;
1837                            sync.behind = 0;
1838                        }
1839                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1840                    }
1841                    self.state.save()?;
1842                }
1843                Phase::Reviewing => {
1844                    self.review_loop_pass().await?;
1845                    if self.resource_halt()? {
1846                        return Ok(());
1847                    }
1848                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1849                    let clean = self.state.reviews.len() > grant.review_start
1850                        && self.state.reviews.last().is_some_and(|r| {
1851                            r.head == head
1852                                && r.clean
1853                                && r.expected > 0
1854                                && r.answered == r.expected
1855                                && matches!(
1856                                    r.e2e_status(),
1857                                    E2eStatus::Passed | E2eStatus::NotConfigured
1858                                )
1859                        });
1860                    if !clean {
1861                        // A confirmation review carries no additional fix budget.
1862                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1863                            == Kind::Review
1864                            && grant.spent < grant.receipt.request.rounds
1865                            && self.state.reviews.last().is_some_and(|r| {
1866                                r.fix.is_none()
1867                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1868                            }) {
1869                            Phase::Pending
1870                        } else {
1871                            Phase::Complete
1872                        };
1873                        self.state.status = RunStatus::Blocked;
1874                        self.state.event(
1875                            "repair-grant",
1876                            "fresh review did not clear the repaired head; no publication",
1877                        );
1878                        self.state.save()?;
1879                        if self.state.repair_grants[index].phase == Phase::Pending {
1880                            continue;
1881                        }
1882                        return Ok(());
1883                    }
1884                    self.state.repair_grants[index].phase = Phase::Gating;
1885                    self.state.save()?;
1886                }
1887                Phase::Gating => {
1888                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1889                    self.state.reconcile_verification_head(&head);
1890                    let wait = self.repository_wait();
1891                    if self
1892                        .state
1893                        .reviews
1894                        .last()
1895                        .is_none_or(|r| r.head != head || !r.clean)
1896                    {
1897                        self.state.status = RunStatus::Blocked;
1898                        self.state.repair_grants[index].phase = Phase::Complete;
1899                        self.state.event(
1900                            "repair-grant",
1901                            "head changed since fresh review; repair held",
1902                        );
1903                        self.state.save()?;
1904                        return Ok(());
1905                    }
1906                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1907                        return Ok(());
1908                    }
1909                    let outcomes = self.run_gate(&winner).await?;
1910                    if verify_inconclusive(&outcomes) {
1911                        return Ok(());
1912                    }
1913                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1914                    if after != head {
1915                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1916                        self.state.repair_grants[index].head = after;
1917                        self.state.repair_grants[index].phase = Phase::Reviewing;
1918                        self.state.status = RunStatus::Blocked;
1919                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1920                        self.state.save()?;
1921                        return Ok(());
1922                    }
1923                    let passed = outcomes.iter().all(CommandOutcome::ok);
1924                    // Follow-up grants owe the same guarded checkout delivery
1925                    // as the conflict repair that introduced the new head.
1926                    if passed
1927                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1928                    {
1929                        return Ok(());
1930                    }
1931                    self.state.repair_grants[index].succeeded = passed;
1932                    self.state.gate = outcomes;
1933                    self.state.gate_ran = true;
1934                    self.state.status = if passed {
1935                        RunStatus::Gating
1936                    } else {
1937                        RunStatus::Blocked
1938                    };
1939                    self.state.repair_grants[index].phase = if !passed
1940                        && grant.receipt.request.kind == Kind::Gate
1941                        && grant.spent < grant.receipt.request.rounds
1942                    {
1943                        Phase::Pending
1944                    } else {
1945                        Phase::Complete
1946                    };
1947                    if passed && self.state.config.merge.mode == MergeMode::Pr {
1948                        // Persist publication debt with the successful checkpoint.
1949                        self.state.pr_publication_pending = true;
1950                    } else if passed {
1951                        self.state.status = RunStatus::Ready;
1952                    }
1953                    self.state.save()?;
1954                    if !passed {
1955                        continue;
1956                    }
1957                    // Existing PR ownership and push lease stay in place. The
1958                    // normal publisher owes the new reviewed/gated SHA, and
1959                    // land's fresh owner approval remains mandatory.
1960                    if self.state.config.merge.mode != MergeMode::Pr {
1961                        self.state.status = RunStatus::Ready;
1962                        self.state.event(
1963                            "repair-grant",
1964                            "repair verified locally; grant carries no merge authorization",
1965                        );
1966                        self.state.save()?;
1967                        return Ok(());
1968                    }
1969                    return self.merge().await;
1970                }
1971                Phase::Complete => return Ok(()),
1972            }
1973        }
1974    }
1975
1976    async fn execute_graph(&mut self) -> Result<()> {
1977        // Even a terminal run may reenter through an explicit recovery action.
1978        // Restore before prep shortcuts, saved conversations or graph nodes use
1979        // a disposable path; GC may already have removed its original objects.
1980        let archive_home = crate::run::home().canonicalize()?;
1981        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1982            let repo = self.state.repo.clone();
1983            let _coordination = self.coordinate(&repo).await?;
1984            ensure_setup_config(&mut self.state, &repo).await?;
1985            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1986        }
1987        self.state.configure_recovery_resume()?;
1988        if self
1989            .state
1990            .repair_grants
1991            .last()
1992            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1993        {
1994            return self.execute_granted_repair().await;
1995        }
1996        if self
1997            .state
1998            .repair_grants
1999            .last()
2000            .is_some_and(|g| !g.succeeded)
2001        {
2002            self.state.status = RunStatus::Blocked;
2003            self.state.save()?;
2004            return Ok(());
2005        }
2006        // Preservation may invalidate an old empty classification. Resolve it
2007        // before any resume shortcut can reuse a verdict or publish a branch.
2008        self.persist_candidates().await?;
2009        // A saved base stop outranks stage markers, including legacy Voting.
2010        // Recovery never reenters implementation, deliberation or voting.
2011        if self.state.base_sync_blocked() {
2012            self.sync_to_base().await?;
2013            if !self.state.base_sync_blocked()
2014                && self.state.status != RunStatus::AlreadyInBase
2015                && !self.resource_halt()?
2016            {
2017                self.finish_after_tally().await?;
2018            }
2019            return Ok(());
2020        }
2021        if self
2022            .state
2023            .merge_decision
2024            .as_ref()
2025            .is_some_and(|decision| !decision.closed)
2026            && self.state.status != RunStatus::Landing
2027        {
2028            if crate::merge_decision::resumable_hold(&self.state) {
2029                self.state.merge = Some(MergeOutcome {
2030                    mode: MergeMode::Pr,
2031                    ok: false,
2032                    detail: self
2033                        .state
2034                        .merge_decision
2035                        .as_ref()
2036                        .unwrap()
2037                        .target
2038                        .pr
2039                        .clone(),
2040                    empty: false,
2041                });
2042                self.run_land().await?;
2043                self.settle_questions();
2044                return Ok(());
2045            }
2046            bail!(
2047                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2048            );
2049        }
2050        if self.state.config.graph.selected_implementer.is_some()
2051            && self.state.status == RunStatus::Blocked
2052            && self.state.tally.is_none()
2053            && self.state.candidate_preservation.is_empty()
2054            && self.state.interrupted_implementations.is_empty()
2055        {
2056            bail!(
2057                "selected implementation is held; explicitly release the owning task to retry it"
2058            );
2059        }
2060
2061        if self.state.merge_repairs.last().is_some_and(|r| {
2062            matches!(
2063                r.phase,
2064                crate::run::MergeRepairPhase::Requested
2065                    | crate::run::MergeRepairPhase::Fixing
2066                    | crate::run::MergeRepairPhase::Reviewing
2067            )
2068        }) {
2069            if self.repair_merge_review().await? {
2070                self.run_land().await?;
2071            }
2072            self.settle_questions();
2073            return Ok(());
2074        }
2075        // Publication already passed review and gate. Resume only that node:
2076        // syncing to a newer base here could change the verified commit.
2077        if self.state.pr_publication_pending {
2078            self.merge().await?;
2079            if self.resource_halt()? {
2080                return Ok(());
2081            }
2082            self.state.save()?;
2083            self.settle_questions();
2084            return Ok(());
2085        }
2086        // A run that already lost its quorum never resumes into the verdict
2087        // machinery: `deliberate` and `vote` would otherwise clobber the
2088        // stalled marker back to Voting and the run would keep going past a
2089        // verdict that is no longer trustworthy. Everything already recorded is
2090        // kept, so the run stays resumable (or foldable) for a human to pick up.
2091        //
2092        // On --resume the run gets one chance to repair itself: the seats a
2093        // rate limit took out are re-asked. If their quota has since reset and
2094        // the quorum is restored, the run picks up and finishes; otherwise it
2095        // stays stale and still-resumable for a later retry. If it does not
2096        // recover, the returned status stays `Stalled` and nothing was
2097        // clobbered (the recovery only mutates entries for the lost seats).
2098        if self.state.status == RunStatus::Stalled {
2099            if self.recover_stall().await? {
2100                self.finish_after_tally().await?;
2101            } else {
2102                // Still below quorum: persist the marker and stay resumable.
2103                self.state.save()?;
2104            }
2105            return Ok(());
2106        }
2107        // A run parked inside `land` - watching CI, mid fix-round, or
2108        // waiting on the owner's merge approval - resumes directly into it,
2109        // never back through `prep`. Everything before `merge` already
2110        // concluded; that is the only way `status` reaches `Landing` in the
2111        // first place. Re-walking `review_loop` first would also be actively
2112        // wrong: its own status recomputation (see its doc) treats any
2113        // clean round as reason to set `status` to `Gating`, which would
2114        // clobber this marker before `merge` ever ran, and this run would
2115        // never find its way back into `land` at all.
2116        if self.state.status == RunStatus::Landing
2117            || crate::handover::landing_checkpoint(&self.state)
2118        {
2119            self.state.land_sync_pending = true;
2120            self.state.save()?;
2121            self.run_land().await?;
2122            // `run_land` may have settled the run right here - CI came back
2123            // green and the PR merged, say - without ever passing back
2124            // through `merge`'s own trailing call. Whatever it left `status`
2125            // as is what this has to read.
2126            self.settle_questions();
2127            return Ok(());
2128        }
2129        // A run parked at `merge` on the owner's word about a withheld PR text
2130        // resumes straight into `merge`. Re-walking the tail would sync to the
2131        // base again (possibly rebasing, possibly spending the fixer) after the
2132        // gate had passed, and then push a tree nobody reviewed or gated.
2133        if self.state.github_text.is_some()
2134            && self.state.merge.is_none()
2135            && self.state.status == RunStatus::Gating
2136        {
2137            self.merge().await?;
2138            if self.resource_halt()? {
2139                return Ok(());
2140            }
2141            self.state.save()?;
2142            return Ok(());
2143        }
2144        if let Err(e) = self.prep().await {
2145            if e.is::<crate::repo_lock::ResourceBlocked>() {
2146                return Err(e);
2147            }
2148            self.state.status = RunStatus::Blocked;
2149            self.state
2150                .prep_failure
2151                .get_or_insert_with(|| format!("{e:#}"));
2152            self.state
2153                .event("prep", format!("preparation stopped: {e:#}"));
2154            self.state.save()?;
2155            return Err(e);
2156        }
2157        if self.park_here()? {
2158            return Ok(());
2159        }
2160        self.advise().await?;
2161        if self.park_here()? {
2162            return Ok(());
2163        }
2164        self.implement().await?;
2165        if self.park_here()? {
2166            return Ok(());
2167        }
2168        // `after_implement` already saved the state and settled any open
2169        // questions when it set this; nothing later in the graph has
2170        // anything to judge.
2171        if self.state.status == RunStatus::VerifiedNoop {
2172            return Ok(());
2173        }
2174        self.judge().await?;
2175        if self.park_here()? {
2176            return Ok(());
2177        }
2178        self.deliberate().await?;
2179        if self.park_here()? {
2180            return Ok(());
2181        }
2182        self.vote().await?;
2183        if self.park_here()? {
2184            return Ok(());
2185        }
2186        self.tally()?;
2187        // A verdict that lost its quorum is not trustworthy: do not review,
2188        // gate, or merge on it. Everything already done is kept, so the run
2189        // stays resumable (or foldable); the human can replace the agent that
2190        // ran out of quota and pick it up.
2191        if self.state.status == RunStatus::Stalled {
2192            // Persist the stalled marker now — the normal end-of-execute save
2193            // below is below this early return, and without it a resumed run
2194            // would reload a pre-tally status and keep going.
2195            self.state.save()?;
2196            return Ok(());
2197        }
2198        self.finish_after_tally().await?;
2199        Ok(())
2200    }
2201
2202    /// A wave could not start its agents because a shared resource stayed busy
2203    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2204    /// resumable. Unstarted seats have no result; completed peers retain
2205    /// their saved candidate state.
2206    fn stop_for_resource(&mut self) -> Result<()> {
2207        // Resource acquisition observes cancellation inside a node. Complete
2208        // the same durable park as every other node boundary before returning.
2209        if self.pause.parked() || self.interrupt.parked() {
2210            self.park_here()?;
2211            return Ok(());
2212        }
2213        if self.state.resource_block.is_some() {
2214            self.state.status = RunStatus::Blocked;
2215        }
2216        self.state.save()
2217    }
2218
2219    /// Stop walking the graph if a node ended on a resource block.
2220    fn resource_halt(&mut self) -> Result<bool> {
2221        if self.state.resource_block.is_none() {
2222            return Ok(false);
2223        }
2224        self.stop_for_resource()?;
2225        Ok(true)
2226    }
2227
2228    /// Park here if asked to, recording it in the run's own timeline.
2229    ///
2230    /// Returns whether the caller should stop walking the graph. The state is
2231    /// saved either way by the node that just finished; this adds the event so
2232    /// the operator's card says why a run that is neither finished nor moving
2233    /// is sitting where it is.
2234    fn park_here(&mut self) -> Result<bool> {
2235        let parking = self.pause.parked() || self.interrupt.parked();
2236        if !parking && self.state.resource_block.is_some() {
2237            self.stop_for_resource()?;
2238            return Ok(true);
2239        }
2240        // Either handle asking is enough - see `Pause`'s own doc for why
2241        // they are never the same one. `interrupt` is checked second so a
2242        // reason it carries is preferred in the message below over a plain
2243        // shutdown park racing it at the same boundary.
2244        if !self.pause.parked() && !self.interrupt.parked() {
2245            return Ok(false);
2246        }
2247        if self.state.parked {
2248            return Ok(true);
2249        }
2250        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2251        if let Some(block) = &mut self.state.resource_block
2252            && matches!(
2253                block.cause,
2254                crate::run::ResourceBlockCause::Cancelled { .. }
2255            )
2256        {
2257            block.cause = crate::run::ResourceBlockCause::Cancelled {
2258                park: cause.clone(),
2259            };
2260        }
2261        let why = match cause.as_ref().map(ParkCause::label) {
2262            Some(reason) => format!(
2263                "parked after `{}` ({reason}) — resume to carry on from here",
2264                self.state.status.as_str()
2265            ),
2266            None => format!(
2267                "parked after `{}` — resume to carry on from here",
2268                self.state.status.as_str()
2269            ),
2270        };
2271        self.state.event("park", why);
2272        self.state
2273            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2274        self.state.parked = true;
2275        self.state.save()?;
2276        Ok(true)
2277    }
2278
2279    /// Only a new explicit operator approval may retry failed preparation.
2280    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2281        if self.state.prep_failure.is_none() {
2282            return Ok(());
2283        }
2284        let failed_at = self
2285            .state
2286            .events
2287            .iter()
2288            .rev()
2289            .find(|event| event.node == "prep")
2290            .map_or(self.state.updated_at, |event| event.at);
2291        anyhow::ensure!(
2292            approval_at > failed_at,
2293            "preparation retry requires a new explicit release after the failure"
2294        );
2295        self.state.prep_failure = None;
2296        self.state.prep_complete = Some(false);
2297        self.state
2298            .event("prep-retry", "operator approved retry of preparation");
2299        Ok(())
2300    }
2301
2302    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2303    pub fn on_pause(&mut self, pause: Pause) {
2304        self.pause = pause;
2305    }
2306
2307    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2308    /// scheduler asking this one run - and no other - to park so a task
2309    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2310    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2311    /// handle.
2312    pub fn watch_interrupt(&mut self, pause: Pause) {
2313        self.interrupt = pause;
2314    }
2315
2316    /// Abandon this run's own open questions, once `status` has actually
2317    /// settled rather than merely paused.
2318    ///
2319    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2320    /// either back up with the candidates, the review round and the seat
2321    /// sessions already on disk, so a question an implementer asked mid-round
2322    /// may still get a real answer read by a real resume. Only the statuses
2323    /// `resumable` excludes are actually final: the run merged, it reached
2324    /// `Ready` with nothing left to do, it failed outright with no
2325    /// established point to continue from, or every candidate agreed, with
2326    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2327    /// every one of those the seat that asked is gone for good, exactly like
2328    /// the run being deleted under `magi run rm` - so the same cleanup
2329    /// applies, worded for what actually happened instead of "the run was
2330    /// deleted".
2331    ///
2332    /// Best-effort and silent on success: called from every place `status`
2333    /// can land on one of those three, including ones a resumed run revisits,
2334    /// so it must cost nothing when there was nothing open to begin with.
2335    fn settle_questions(&mut self) {
2336        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2337            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2338        }
2339    }
2340
2341    /// The tail of the graph after a trustworthy tally: fold losers, review,
2342    /// gate, merge, and persist.
2343    async fn finish_after_tally(&mut self) -> Result<()> {
2344        // Before the losers are folded: their branches are what the graft
2345        // reads from.
2346        self.graft().await?;
2347        if self.resource_halt()? {
2348            return Ok(());
2349        }
2350        self.fold_losers().await?;
2351        // Before review starts, and again right before the gate: a run's
2352        // review rounds can themselves take long enough for the base to move
2353        // a second time, and the gate is the one node whose "green" gets
2354        // acted on.
2355        self.sync_to_base().await?;
2356        if self.hold_base_sync().await? || self.resource_halt()? {
2357            return Ok(());
2358        }
2359        if self.state.status == RunStatus::AlreadyInBase {
2360            return Ok(());
2361        }
2362        self.review_loop().await?;
2363        if self.resource_halt()? {
2364            return Ok(());
2365        }
2366        self.sync_to_base().await?;
2367        if self.hold_base_sync().await? || self.resource_halt()? {
2368            return Ok(());
2369        }
2370        if self.state.status == RunStatus::AlreadyInBase {
2371            return Ok(());
2372        }
2373        self.gate().await?;
2374        self.merge().await?;
2375        if self.resource_halt()? {
2376            return Ok(());
2377        }
2378        self.state.save()?;
2379        Ok(())
2380    }
2381
2382    // ---------------------------------------------------------------- prep
2383
2384    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2385    /// run with the step and its output in the event; it is never swallowed.
2386    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2387    /// resume starts over instead of finding candidates and missing seats.
2388    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2389        let result = match ensure_setup_config(&mut self.state, repo).await {
2390            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2391            Err(e) => Err(e),
2392        };
2393        if let Err(e) = result {
2394            git::worktree_remove_clean(repo, wt).await.ok();
2395            if in_prep {
2396                self.rollback_prep(repo).await;
2397            }
2398            self.state.status = RunStatus::Blocked;
2399            self.state.event(
2400                "setup",
2401                format!("worktree setup failed in {}: {e:#}", wt.display()),
2402            );
2403            self.state.save()?;
2404            return Err(e);
2405        }
2406        Ok(())
2407    }
2408
2409    /// Undo everything `prep` created, so a resume re-runs it whole.
2410    async fn rollback_prep(&mut self, repo: &Path) {
2411        let root = self.state.worktree_root();
2412        for c in std::mem::take(&mut self.state.candidates) {
2413            if matches!(
2414                git::worktree_remove_clean(repo, &c.worktree).await,
2415                Ok(true)
2416            ) {
2417                git::branch_delete(repo, &c.branch).await.ok();
2418            } else {
2419                self.state.candidates.push(c);
2420            }
2421        }
2422        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2423        for j in 1..=seats {
2424            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2425                .await
2426                .ok();
2427        }
2428        for k in 1..=self.state.config.graph.advisors {
2429            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2430                .await
2431                .ok();
2432        }
2433    }
2434
2435    async fn prep(&mut self) -> Result<()> {
2436        if let Some(reason) = &self.state.prep_failure {
2437            bail!(
2438                "candidate preparation previously failed: {reason}; correct the source before retrying"
2439            );
2440        }
2441        if self
2442            .state
2443            .prep_complete
2444            .unwrap_or(!self.state.candidates.is_empty())
2445        {
2446            return Ok(());
2447        }
2448        let repo = self.state.repo.clone();
2449        // Includes rollback, config, refs, checkout creation and setup. No
2450        // repository mutation may precede the real shared lock acquisition.
2451        let _coordination = self.coordinate(&repo).await?;
2452        let base = self.state.base_commit.clone();
2453        let plan = self.state.sources.plan(&repo, &base).await?;
2454        self.rollback_prep(&repo).await;
2455        if !self.state.candidates.is_empty() {
2456            bail!(
2457                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2458            );
2459        }
2460        self.state.prep_complete = Some(false);
2461        self.state.status = RunStatus::Prep;
2462        self.state.save()?;
2463        let repo = self.state.repo.clone();
2464        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2465        let root = self.state.worktree_root();
2466        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2467
2468        // The hook is the write-time half of the blindness contract; the
2469        // presentation filter in `blind` is the half that cannot be bypassed.
2470        let hooks_dir = self.state.dir().join("hooks");
2471        if self.state.config.blind.commit_msg_hook {
2472            std::fs::create_dir_all(&hooks_dir)
2473                .with_context(|| format!("create {}", hooks_dir.display()))?;
2474            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2475            let path = hooks_dir.join("commit-msg");
2476            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2477            make_executable(&path)?;
2478            // Ref-counted rather than a plain idempotent set: with more than
2479            // one run able to be in flight in the same repository at once
2480            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2481            // true?" check cannot tell "another run of mine still needs
2482            // this" from "nobody does", and the run that happens to finish
2483            // first would disable the hook out from under a sibling still
2484            // relying on it.
2485            git::acquire_worktree_config(&repo).await?;
2486            self.state.enabled_worktree_config = true;
2487        }
2488
2489        // `[worktree] setup` hides its products per worktree, which needs the
2490        // same per-worktree config the hook does. Held until fold, like it.
2491        ensure_setup_config(&mut self.state, &repo).await?;
2492
2493        for (index, (spec, label)) in self
2494            .roles
2495            .implementers
2496            .clone()
2497            .into_iter()
2498            .zip(labels)
2499            .enumerate()
2500        {
2501            let branch = self.state.branch_for(label);
2502            let worktree = root.join(format!("cand-{label}"));
2503            // Persist intent before Git can leave a partially created tree.
2504            // Candidate registration follows setup, so it cannot inventory
2505            // crashes or refusals during creation itself.
2506            self.state.event(
2507                "prep-intent",
2508                serde_json::json!({
2509                    "branch": branch,
2510                    "worktree": worktree,
2511                    "start": start,
2512                    "picks": plan.picks,
2513                })
2514                .to_string(),
2515            );
2516            self.state.save()?;
2517            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2518            if self.state.config.blind.commit_msg_hook {
2519                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2520            }
2521            git::local_exclude(&worktree, "/.magi/").await?;
2522            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2523                // Nothing of this prep may survive: a resume must start over,
2524                // not find half the candidates and skip the rest.
2525                git::worktree_remove_clean(&repo, &worktree).await.ok();
2526                git::branch_delete(&repo, &branch).await.ok();
2527                self.rollback_prep(&repo).await;
2528                self.state.status = RunStatus::Blocked;
2529                self.state
2530                    .event("prep", format!("worktree setup failed: {e:#}"));
2531                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2532                self.state.save()?;
2533                return Err(e);
2534            }
2535            self.state.candidates.push(Candidate {
2536                index,
2537                label,
2538                agent: spec.id.clone(),
2539                branch,
2540                worktree,
2541                summary: String::new(),
2542                stat: String::new(),
2543                files: 0,
2544                commits: 0,
2545                empty: false,
2546                failed: None,
2547                verified_noop: None,
2548                duration_ms: 0,
2549                folded: false,
2550            });
2551            self.state.save()?;
2552            for pick in &plan.picks {
2553                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2554                {
2555                    self.state.status = RunStatus::Blocked;
2556                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2557                    self.state
2558                        .event("prep", format!("cannot apply referenced commit: {e}"));
2559                    self.state.save()?;
2560                    return Err(e);
2561                }
2562            }
2563        }
2564
2565        for j in 1..=self.roles.judges.len() {
2566            let wt = root.join(format!("judge-{j}"));
2567            if !wt.exists() {
2568                git::worktree_add_detached(&repo, &wt, &base).await?;
2569                self.setup_seat_worktree(&repo, &wt, true).await?;
2570            }
2571        }
2572
2573        // Disposable, detached checkouts for the design-deliberation stage's
2574        // advisor seats — the same shape as the judges' above, at the same
2575        // base commit, since advisors also only ever read. Sized off the
2576        // configured count directly rather than a resolved roster: unlike
2577        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2578        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2579        // `prep` has no `ResolvedRoles` field to read a count from here.
2580        if self.state.config.graph.advise {
2581            for k in 1..=self.state.config.graph.advisors {
2582                let wt = root.join(format!("advisor-{k}"));
2583                if !wt.exists() {
2584                    git::worktree_add_detached(&repo, &wt, &base).await?;
2585                    self.setup_seat_worktree(&repo, &wt, true).await?;
2586                }
2587            }
2588        }
2589
2590        // A judge cannot tell it is looking at its own patch — the seats keep
2591        // separate conversations — but a panel that shares agents with the
2592        // field is less independent than it looks, and that is worth saying out
2593        // loud once per run rather than leaving it in the config.
2594        let authors: Vec<&str> = self
2595            .roles
2596            .implementers
2597            .iter()
2598            .map(|a| a.id.as_str())
2599            .collect();
2600        let overlap: Vec<String> = self
2601            .roles
2602            .judges
2603            .iter()
2604            .enumerate()
2605            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2606            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2607            .collect();
2608        if !overlap.is_empty() {
2609            let note = format!(
2610                "{} also authored a candidate; blind, but the panel is less \
2611                 independent than {} distinct agents would be",
2612                overlap.join(", "),
2613                self.roles.judges.len()
2614            );
2615            self.state.event("prep", note);
2616        }
2617
2618        self.state.event(
2619            "prep",
2620            format!(
2621                "{} candidates, {} judges, base {} ({})",
2622                self.state.candidates.len(),
2623                self.roles.judges.len(),
2624                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2625                self.state.base_branch
2626            ),
2627        );
2628        self.state.prep_complete = Some(true);
2629        self.state.status = RunStatus::Implementing;
2630        self.state.save()?;
2631        Ok(())
2632    }
2633
2634    // -------------------------------------------------------------- advise
2635
2636    /// The design-deliberation stage: independent, read-only advisor seats
2637    /// each sketch a design before any implementer touches the repository,
2638    /// and (when at least one produced a usable proposal) a synthesis seat
2639    /// blends them into a brief `implement` carries in every candidate's
2640    /// prompt.
2641    ///
2642    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2643    /// advisors` is the proposal count. Everything here is best-effort and
2644    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2645    /// that cannot reach quota, or a synthesis seat that produced nothing
2646    /// usable all leave `implement` exactly as it was before this stage
2647    /// existed — the task instruction alone — rather than failing the whole
2648    /// competition over an enrichment stage. Every outcome is still recorded
2649    /// as an event, so a run that got nothing from this stage says why.
2650    ///
2651    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2652    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2653    /// resumed run whose stage failed would re-run it, and re-spend the
2654    /// agent calls, on every reentry before `implement`.
2655    ///
2656    /// Also skipped once any candidate shows implementation progress — the
2657    /// exact predicate `implement` itself uses to decide a candidate is no
2658    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2659    /// is not enough: a run created by an older binary that predates this
2660    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2661    /// an already-`Implementing`-or-later run under this build would
2662    /// otherwise walk straight back through `prep` (a no-op once candidates
2663    /// exist) into this node and spawn every advisor seat against worktrees
2664    /// `prep` never recreated — after implementation has already started,
2665    /// which is exactly the invariant this stage exists to guarantee.
2666    async fn advise(&mut self) -> Result<()> {
2667        let implement_untouched = self
2668            .state
2669            .candidates
2670            .iter()
2671            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2672        if !self.state.config.graph.advise || self.state.advise_attempted {
2673            return Ok(());
2674        }
2675        if !implement_untouched {
2676            self.state.event(
2677                "advise",
2678                "skipping the design-deliberation stage: at least one \
2679                 candidate already shows implementation progress, so this \
2680                 run is past the point the stage exists to run before"
2681                    .to_owned(),
2682            );
2683            self.state.advise_attempted = true;
2684            self.state.save()?;
2685            return Ok(());
2686        }
2687        let run_id = self.state.id.clone();
2688        let prompts = self.state.config.prompts.clone();
2689        let instruction = self.state.instruction.clone();
2690        let language = self.state.config.graph.language.clone();
2691        let root = self.state.worktree_root();
2692        let n = self.state.config.graph.advisors;
2693        let where_recorded = self.state.dir().join("run.json");
2694
2695        let seats = match self.state.config.advisors() {
2696            Ok(seats) if !seats.is_empty() => seats,
2697            Ok(_) => {
2698                self.state.event(
2699                    "advise",
2700                    format!(
2701                        "[graph] advisors is 0; skipping the design-deliberation \
2702                         stage and continuing without a synthesis brief (see {})",
2703                        where_recorded.display()
2704                    ),
2705                );
2706                self.state.advise_attempted = true;
2707                self.state.save()?;
2708                return Ok(());
2709            }
2710            Err(e) => {
2711                self.state.event(
2712                    "advise",
2713                    format!(
2714                        "could not resolve advisor seats ({e:#}); continuing \
2715                         without a design-deliberation brief (see {})",
2716                        where_recorded.display()
2717                    ),
2718                );
2719                self.state.advise_attempted = true;
2720                self.state.save()?;
2721                return Ok(());
2722            }
2723        };
2724
2725        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2726        let artifacts = agent::artifacts_dir(&self.state.dir());
2727        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2728
2729        let mut jobs = Vec::new();
2730        for (i, spec) in seats.iter().cloned().enumerate() {
2731            let seat_key = format!("advisor-{}", i + 1);
2732            let seat = self.seat(&seat_key, &spec.id);
2733            jobs.push(SeatJob {
2734                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2735                spec,
2736                seat,
2737                cwd: worktrees[i % worktrees.len()].clone(),
2738                timeout,
2739                allow_write: false,
2740                sessions: false,
2741                artifacts: artifacts.clone(),
2742                stem: seat_key,
2743                handover: None,
2744            });
2745        }
2746
2747        self.state.event(
2748            "advise",
2749            format!(
2750                "{} advisor seat(s) sketching a design in parallel",
2751                jobs.len()
2752            ),
2753        );
2754        let mut quota_losses = Vec::new();
2755        let cache = self.state.config.cache_dir();
2756        let ctx = WaveCtx {
2757            carry_seats: false,
2758            pauses: Some((&self.pause, &self.interrupt)),
2759            run: &run_id,
2760            node: "advise",
2761            prompts: &prompts,
2762            cache: cache.as_deref(),
2763            round: None,
2764        };
2765        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2766        let results = ask_json_wave::<Proposal>(
2767            jobs,
2768            Arc::clone(&self.sem),
2769            self.state.config.graph.retries,
2770            &advisor_roster,
2771            &ctx,
2772            &mut quota_losses,
2773            &mut self.state,
2774            &|p: &Proposal| p.validate(),
2775        )
2776        .await;
2777        if self.state.resource_block.is_some() {
2778            return self.stop_for_resource();
2779        }
2780        self.state.quota.extend(quota_losses);
2781
2782        let mut records = Vec::with_capacity(results.len());
2783        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2784            let agent_id = seat.agent.clone();
2785            self.state.seats.insert(seat.key.clone(), seat);
2786            match res {
2787                Ok((proposal, out)) => {
2788                    self.state
2789                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2790                    records.push(advise::AdvisorRecord::proposed(
2791                        i + 1,
2792                        agent_id,
2793                        proposal,
2794                        out.duration_ms,
2795                    ));
2796                }
2797                Err(e) => {
2798                    self.state.event(
2799                        "advise",
2800                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2801                    );
2802                    records.push(advise::AdvisorRecord::failed(
2803                        i + 1,
2804                        agent_id,
2805                        e.to_string(),
2806                    ));
2807                }
2808            }
2809        }
2810
2811        let mut advice = advise::Advice {
2812            records,
2813            synthesis: None,
2814        };
2815        if advice.proposals().is_empty() {
2816            self.state.event(
2817                "advise",
2818                "no advisor produced a usable proposal; continuing without a \
2819                 synthesis brief"
2820                    .to_owned(),
2821            );
2822        } else {
2823            match self
2824                .synthesize_brief(
2825                    &advice,
2826                    &instruction,
2827                    &language,
2828                    &worktrees[0],
2829                    &artifacts,
2830                    &run_id,
2831                    &prompts,
2832                    cache.as_deref(),
2833                )
2834                .await
2835            {
2836                Ok(Some(text)) => {
2837                    self.state.event(
2838                        "advise",
2839                        "synthesized a design brief for the implementer".to_owned(),
2840                    );
2841                    advice.synthesis = Some(text);
2842                }
2843                Ok(None) => {
2844                    self.state.event(
2845                        "advise",
2846                        "the synthesis seat produced nothing usable; continuing \
2847                         without a design brief"
2848                            .to_owned(),
2849                    );
2850                }
2851                Err(e) => {
2852                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2853                        return Err(e);
2854                    }
2855                    self.state.event(
2856                        "advise",
2857                        format!("could not synthesize a design brief: {e:#}"),
2858                    );
2859                }
2860            }
2861        }
2862        advise::apply_reflection(&mut advice);
2863
2864        self.state.advice = Some(advice);
2865        self.state.advise_attempted = true;
2866        self.state.save()?;
2867        Ok(())
2868    }
2869
2870    /// The synthesis seat: reads every advisor's proposal and blends them
2871    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2872    /// out of [`Runner::advise`] only for readability — it is not called
2873    /// anywhere else.
2874    ///
2875    /// Picked the same way [`crate::talk`]'s standing conversation and
2876    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2877    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2878    /// order (a claude seat, else the first runnable agent in roster order)
2879    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2880    /// in [`crate::config`] for why a dedicated field exists here at all.
2881    #[allow(clippy::too_many_arguments)]
2882    async fn synthesize_brief(
2883        &mut self,
2884        advice: &advise::Advice,
2885        instruction: &str,
2886        language: &str,
2887        cwd: &Path,
2888        artifacts: &Path,
2889        run_id: &str,
2890        prompts: &Prompts,
2891        cache: Option<&Path>,
2892    ) -> Result<Option<String>> {
2893        let chain = agent::pick_chain(
2894            &self.state.config.agents,
2895            self.state.config.roles.synthesizer.as_ref(),
2896            &agent::installed,
2897            "synthesizer",
2898        )?;
2899        let proposals = advice.proposals();
2900        let mut prompt = prompt::with_overlay(
2901            prompt::synthesize_brief(instruction, &proposals, language),
2902            prompts.overlay("advise"),
2903        );
2904        if cache.is_some() {
2905            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2906            // below — see `prompt::build_cache_note`'s doc for why telling a
2907            // read-only seat to build through the shared cache is exactly how
2908            // a sandbox's write refusal gets misread as a defect.
2909            prompt.push('\n');
2910            prompt.push_str(&prompt::build_cache_note("advise", false));
2911        }
2912        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2913        // Each id is tried once, in order; a quota hit, error or unusable
2914        // answer moves to the next. The seat is single-turn (`sessions:
2915        // false`) and the prompt is the whole context, so a fallback agent
2916        // needs nothing carried over.
2917        let mut last = None;
2918        for (n, spec) in chain.iter().enumerate() {
2919            if n > 0 {
2920                self.state
2921                    .event("advise", format!("synthesis falling back to {}", spec.id));
2922            }
2923            let mut seat = self.seat("advise-synthesis", &spec.id);
2924            let wait = self.repository_wait();
2925            let outcome = wait
2926                .scope(agent::invoke(
2927                    spec,
2928                    &mut seat,
2929                    &Invocation {
2930                        coordination: self.state.config.coordination.as_ref(),
2931                        fetched_base: None,
2932                        cwd,
2933                        prompt: &prompt,
2934                        timeout,
2935                        allow_write: false,
2936                        sessions: false,
2937                        artifacts,
2938                        stem: &if n == 0 {
2939                            "advise-synthesis".to_owned()
2940                        } else {
2941                            format!("advise-synthesis-{}", spec.id)
2942                        },
2943                        run: run_id,
2944                        node: "advise",
2945                        cache_dir: None,
2946                        attachments: &[],
2947                        writable: &[],
2948                    },
2949                ))
2950                .await;
2951            if outcome.is_ok() {
2952                self.state.seats.insert(seat.key.clone(), seat);
2953            }
2954            let advance = agent::chain_advances(&outcome);
2955            last = Some(outcome);
2956            if !advance {
2957                break;
2958            }
2959        }
2960        // Exhausted: the last attempt's result is what a single failed seat
2961        // would have produced.
2962        let out = last.expect("a chain holds at least one agent")?;
2963        if !out.usable() {
2964            return Ok(None);
2965        }
2966        let text =
2967            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2968        Ok((!text.trim().is_empty()).then_some(text))
2969    }
2970
2971    // ----------------------------------------------------------- implement
2972
2973    async fn implement(&mut self) -> Result<()> {
2974        self.persist_candidates().await?;
2975        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2976        // agent files with `magi task add` name the run that paid for it. The
2977        // prompt overlay is cloned alongside it because the waves borrow it
2978        // while `self` is mutably borrowed by the node's own bookkeeping.
2979        let run_id = self.state.id.clone();
2980        let prompts = self.state.config.prompts.clone();
2981        let todo: Vec<usize> = self
2982            .state
2983            .candidates
2984            .iter()
2985            .enumerate()
2986            .filter(|(_, c)| {
2987                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2988                    && c.failed.is_none()
2989                    && !c.empty
2990            })
2991            .map(|(i, _)| i)
2992            .collect();
2993        if todo.is_empty() {
2994            return self.after_implement();
2995        }
2996        self.state.status = RunStatus::Implementing;
2997
2998        let language = self.state.config.graph.language.clone();
2999        let github_language = self.state.config.graph.github_language.clone();
3000        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3001        let sessions = self.state.config.graph.sessions;
3002        let artifacts = agent::artifacts_dir(&self.state.dir());
3003        // The design-deliberation stage's blended brief, when `advise` found
3004        // one — carried into every implementer's prompt the same way
3005        // regardless of which candidate it is.
3006        let brief = self
3007            .state
3008            .advice
3009            .as_ref()
3010            .and_then(|a| a.synthesis.as_deref())
3011            .map(str::to_owned);
3012        let attachments = self.state.attachments.clone();
3013
3014        let mut jobs = Vec::new();
3015        for &i in &todo {
3016            let (index, label, worktree) = {
3017                let c = &self.state.candidates[i];
3018                (c.index, c.label, c.worktree.clone())
3019            };
3020            let spec = self.roles.implementers[index].clone();
3021            let seat_key = format!("impl-{label}");
3022            let seat = self.seat(&seat_key, &spec.id);
3023            let instruction = seeded_instruction(&self.state);
3024            jobs.push(SeatJob {
3025                spec,
3026                seat,
3027                prompt: prompt::implement_in(
3028                    &instruction,
3029                    &worktree.to_string_lossy(),
3030                    &language,
3031                    &github_language,
3032                    brief.as_deref(),
3033                    &attachments,
3034                ),
3035                cwd: worktree,
3036                timeout,
3037                allow_write: true,
3038                sessions,
3039                artifacts: artifacts.clone(),
3040                stem: format!("impl-{label}"),
3041                handover: None,
3042            });
3043        }
3044
3045        self.state.event(
3046            "implement",
3047            format!("{} candidates in parallel", jobs.len()),
3048        );
3049        // Kept so a seat whose CLI hung up can be asked again from the same
3050        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3051        // can update a seat's own entry once a fallback agent takes it over —
3052        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3053        // whichever agent actually answered, not the one that quota'd out.
3054        let mut sent = jobs.clone();
3055        let cache = self.state.config.cache_dir();
3056        let ctx = WaveCtx {
3057            carry_seats: false,
3058            pauses: Some((&self.pause, &self.interrupt)),
3059            run: &run_id,
3060            node: "implement",
3061            prompts: &prompts,
3062            cache: cache.as_deref(),
3063            round: None,
3064        };
3065        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3066        if !wave_blocked(&results) {
3067            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3068                .await;
3069            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3070                .await?;
3071            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3072                .await;
3073        }
3074        // Do not finalize failed candidates when the entire wave needs machine
3075        // recovery: leaving them pending resumes this node with its work intact.
3076        if !results.is_empty()
3077            && results
3078                .iter()
3079                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3080        {
3081            let losses: Vec<_> = results
3082                .iter()
3083                .map(|(_, seat, out)| {
3084                    let AgentOutcome::Quota(o) = out else {
3085                        unreachable!()
3086                    };
3087                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3088                })
3089                .collect();
3090            self.state.quota.extend(losses.clone());
3091            for (_, seat, _) in results {
3092                self.state.seats.insert(seat.key.clone(), seat);
3093            }
3094            for &i in &todo {
3095                let label = self.state.candidates[i].label;
3096                self.state.interrupted_implementations.insert(label);
3097                self.state
3098                    .candidate_preservation
3099                    .insert(label, "machine outage awaiting persistence".into());
3100            }
3101            // Preserve partial edits too, while keeping every interrupted seat
3102            // pending even when its tree already contains an inherited patch.
3103            block_machine_outage(&mut self.state, "implement", &losses);
3104            self.state.save()?;
3105            self.persist_candidates().await?;
3106            for &i in &todo {
3107                self.state
3108                    .interrupted_implementations
3109                    .insert(self.state.candidates[i].label);
3110            }
3111            return self.stop_for_resource();
3112        }
3113        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3114
3115        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3116            && results
3117                .iter()
3118                .any(|(_, _, out)| FailClass::of(out).is_some());
3119        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3120            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3121                // Executed turns interrupted by disk pressure already carry
3122                // a preservation barrier from wave. Admission-only waits do
3123                // not. Keep the interrupted seat's session for recovery too.
3124                if self
3125                    .state
3126                    .candidate_preservation
3127                    .contains_key(&self.state.candidates[i].label)
3128                {
3129                    self.state.seats.insert(seat.key.clone(), seat);
3130                }
3131                continue;
3132            }
3133            let seat_key = seat.key.clone();
3134            // A quota fallback (`resume_seat_handovers`) may have handed this
3135            // seat to a different agent than the one `prep` recorded on the
3136            // candidate; the stats tables and any later fixer-defaults-to-
3137            // winner's-author lookup must credit whoever actually answered —
3138            // unless every fallback also quota'd out, in which case nobody
3139            // actually answered and crediting the last agent tried would
3140            // erase every earlier agent's own quota loss from the stats
3141            // tables instead of just this one seat's.
3142            let agent = seat.agent.clone();
3143            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3144            self.state.seats.insert(seat.key.clone(), seat);
3145            let label = self.state.candidates[i].label;
3146            self.state.interrupted_implementations.remove(&label);
3147
3148            let (summary, duration, failed, verified_claim) = match out {
3149                AgentOutcome::Ok(o) => {
3150                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3151                    let failed = (!o.usable()).then(|| {
3152                        if o.timed_out {
3153                            "agent timed out".to_owned()
3154                        } else {
3155                            format!(
3156                                "agent exited with {}",
3157                                crate::notices::exit_status(o.exit_code)
3158                            )
3159                        }
3160                    });
3161                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3162                    (text, o.duration_ms, failed, verified_claim)
3163                }
3164                // Left un-resumed by `resume_undelivered` (a dirty tree
3165                // already rescues the work, or there was no session left to
3166                // resume into) — reported like the ordinary failure it is,
3167                // never as if `o.text` (the CLI's raw error JSON) were an
3168                // answer.
3169                AgentOutcome::Dropped(o) => {
3170                    let why = o
3171                        .dropped
3172                        .as_ref()
3173                        .map(|d| d.why.as_str())
3174                        .unwrap_or("the CLI ended the stream without delivering its answer");
3175                    (
3176                        String::new(),
3177                        o.duration_ms,
3178                        Some(format!("the CLI dropped the stream ({why})")),
3179                        None,
3180                    )
3181                }
3182                AgentOutcome::Quota(o) => {
3183                    self.state.quota.push(QuotaLoss::from_report(
3184                        seat_key,
3185                        "implement",
3186                        o.quota.as_ref(),
3187                    ));
3188                    (
3189                        String::new(),
3190                        o.duration_ms,
3191                        Some("rate limited (quota); produced no change".to_owned()),
3192                        None,
3193                    )
3194                }
3195                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3196                    (String::new(), 0, Some(e), None)
3197                }
3198            };
3199
3200            // Save every completed turn before any fallible Git operation. A failed
3201            // rescue must not erase the reply or prevent collection of other seats.
3202            let c = &mut self.state.candidates[i];
3203            if !exhausted_the_fallback_chain {
3204                c.agent = agent;
3205            }
3206            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3207            c.duration_ms = duration;
3208            c.failed = failed;
3209            c.verified_noop = verified_claim;
3210            self.state
3211                .candidate_preservation
3212                .insert(label, "awaiting persistence".to_owned());
3213        }
3214        self.state.save()?;
3215        self.persist_candidates().await?;
3216
3217        if resource_blocked {
3218            return self.stop_for_resource();
3219        }
3220        if selected_failed {
3221            self.state.status = RunStatus::Blocked;
3222            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3223            self.state.save()?;
3224            bail!(
3225                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3226            );
3227        }
3228        self.after_implement()
3229    }
3230
3231    /// Retry only persistence after a completed implementation turn.
3232    async fn persist_candidates(&mut self) -> Result<()> {
3233        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3234        for label in labels {
3235            let i = self
3236                .state
3237                .candidates
3238                .iter()
3239                .position(|c| c.label == label)
3240                .context("pending candidate is missing")?;
3241            let c = self.state.candidates[i].clone();
3242            let interrupted = self.state.interrupted_implementations.contains(&label);
3243            let result = async {
3244                let wait = crate::repo_lock::WaitPolicy::new(
3245                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3246                    || false,
3247                );
3248                let _coordination =
3249                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3250                // Best effort evidence requires no index writes. A successful snapshot
3251                // never authorizes removal of a tree whose rescue failed.
3252                if let Err(e) = git::snapshot_uncommitted(
3253                    &c.worktree,
3254                    &self
3255                        .state
3256                        .dir()
3257                        .join("artifacts")
3258                        .join(format!("candidate-{label}-recovery")),
3259                )
3260                .await
3261                {
3262                    self.state.event(
3263                        "preserve",
3264                        format!("candidate {label} snapshot incomplete: {e:#}"),
3265                    );
3266                }
3267                let rescued = git::rescue_commit(
3268                    &c.worktree,
3269                    &format!("magi: candidate {label} (uncommitted work)"),
3270                )
3271                .await?;
3272                self.state.note_withheld("implement", &rescued.withheld);
3273                let base = &self.state.base_commit;
3274                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3275                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3276                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3277                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3278                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3279                if c.empty && commits > 0 && !patch.trim().is_empty()
3280                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3281                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3282                        || self.state.tally.is_some())
3283                {
3284                    // Keep the old classification and pending marker so every
3285                    // retry stops too. Retain decisions as recovery evidence;
3286                    // only a supported recovery operation may replace them.
3287                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3288                }
3289                let c = &mut self.state.candidates[i];
3290                c.commits = commits;
3291                c.stat = stat;
3292                c.files = files;
3293                let no_changes = commits == 0 || patch.trim().is_empty();
3294                c.empty = no_changes && !interrupted;
3295                if interrupted && no_changes {
3296                    c.failed = None;
3297                    c.verified_noop = None;
3298                }
3299                if !c.empty {
3300                    c.failed = None;
3301                    c.verified_noop = None;
3302                }
3303                self.state.event(
3304                    "preserve",
3305                    format!("candidate {label}: {files} files, {commits} commits"),
3306                );
3307                Ok::<_, anyhow::Error>(())
3308            }
3309            .await;
3310            match result {
3311                Ok(()) => {
3312                    self.state.candidate_preservation.remove(&label);
3313                    if !interrupted || self.state.candidates[i].files > 0 {
3314                        self.state.interrupted_implementations.remove(&label);
3315                    }
3316                }
3317                Err(e) => {
3318                    let reason = format!("candidate {label} preservation failed: {e:#}");
3319                    self.state.event("preserve", &reason);
3320                    self.state.candidate_preservation.insert(label, reason);
3321                }
3322            }
3323            if self.state.candidate_preservation.is_empty()
3324                && self.state.status == RunStatus::Blocked
3325                && self.state.candidates.iter().all(|c| c.failed.is_none())
3326            {
3327                // Persist the recovered phase with the cleared pending marker;
3328                // a crash between saves must not turn this into a selected-seat
3329                // failure hold on the next resume.
3330                self.state.status = RunStatus::Implementing;
3331                self.state.event(
3332                    "preserve",
3333                    "candidate persistence recovered; implementation can continue",
3334                );
3335            }
3336            self.state.save()?;
3337        }
3338        if !self.state.candidate_preservation.is_empty() {
3339            self.state.status = RunStatus::Blocked;
3340            self.state.save()?;
3341            bail!(
3342                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3343            );
3344        }
3345        Ok(())
3346    }
3347
3348    /// Ask again, once, for work a CLI did and then failed to hand over.
3349    ///
3350    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3351    /// status with an empty response and a usage report showing output tokens,
3352    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3353    /// seven minutes and 14,267 output tokens that arrived as an empty
3354    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3355    ///
3356    /// Two conditions, and both matter:
3357    ///
3358    /// - **Only when the tree is untouched.** Often the agent has already
3359    ///   written its files and only the closing message was lost; the rescue
3360    ///   commit below picks that up and there is nothing to ask for. Re-asking
3361    ///   then would pay for a second implementation of work already on disk.
3362    /// - **Once.** A CLI that drops one stream can drop the next, and this
3363    ///   node is the most expensive in the graph.
3364    ///
3365    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3366    /// dropped reply still carried its `conversation_id`, so the seat is asked
3367    /// to finish what it was doing rather than sent the whole task again. It
3368    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3369    /// node's - for the same reason a re-ranked judge does: restating finished
3370    /// work is not the work.
3371    ///
3372    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3373    /// same way until it resets, while an abandoned conversation is still
3374    /// there to be picked up.
3375    async fn resume_undelivered(
3376        &mut self,
3377        results: &mut [(usize, SeatState, AgentOutcome)],
3378        sent: &[SeatJob],
3379        prompts: &Prompts,
3380        run_id: &str,
3381    ) {
3382        for (wi, seat, out) in results.iter_mut() {
3383            let Some(dropped) = (match &*out {
3384                AgentOutcome::Dropped(o) => o.dropped.clone(),
3385                _ => None,
3386            }) else {
3387                continue;
3388            };
3389            let Some(job) = sent.get(*wi) else { continue };
3390            // Already on disk? Then only the closing message was lost.
3391            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3392                self.state.event(
3393                    "implement",
3394                    format!(
3395                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3396                         work is in the tree",
3397                        seat.key, dropped.output_tokens, dropped.why
3398                    ),
3399                );
3400                continue;
3401            }
3402            // The re-ask only makes sense as a resume: `resume_after_drop`
3403            // says nothing about the task, trusting the seat to still hold it.
3404            // Without a session to resume — sessions disabled, or this CLI's
3405            // drop shape happened not to carry a session id — that prompt
3406            // would open a brand-new conversation with no context at all,
3407            // which is worse than leaving this as the ordinary failure it
3408            // already is.
3409            if !has_context(&job.spec, seat, job.sessions) {
3410                self.state.event(
3411                    "implement",
3412                    format!(
3413                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3414                         is no session left to resume",
3415                        seat.key, dropped.output_tokens, dropped.why
3416                    ),
3417                );
3418                continue;
3419            }
3420            self.state.event(
3421                "implement",
3422                format!(
3423                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3424                     conversation",
3425                    seat.key, dropped.output_tokens, dropped.why
3426                ),
3427            );
3428            let mut retry = job.clone();
3429            retry.seat = seat.clone();
3430            retry.prompt = prompt::resume_after_drop(&dropped.why);
3431            retry.timeout = retry_budget(job.timeout, true);
3432            retry.stem = format!("{}-resume", job.stem);
3433            let cache = self.state.config.cache_dir();
3434            let ctx = WaveCtx {
3435                carry_seats: false,
3436                pauses: Some((&self.pause, &self.interrupt)),
3437                run: run_id,
3438                node: "implement",
3439                prompts,
3440                cache: cache.as_deref(),
3441                round: None,
3442            };
3443            let (resumed_seat, resumed) =
3444                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3445            *seat = resumed_seat;
3446            *out = resumed;
3447        }
3448    }
3449
3450    /// Fall an implement seat through to the next untried agent in the
3451    /// implementer roster when it lost to quota — or, since the handover was
3452    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3453    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3454    /// quota path itself being unchanged — instead of leaving the
3455    /// seat's loss final the moment one agent's account runs dry.
3456    ///
3457    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3458    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3459    /// truncates to the single slot rotation picked, so a solo task whose one
3460    /// implementer hits quota mid-run used to have nothing else to try. This
3461    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3462    /// unrotated roster — which is the only place the *other* candidates in
3463    /// the machine's roster still exist once `implementers` has been cut down
3464    /// to size.
3465    ///
3466    /// Walks forward from just past the seat's own original position in the
3467    /// roster, never wrapping back to the front: a later candidate slot (say
3468    /// `beta`, the roster's second entry) must fall through to the *next*
3469    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3470    /// almost certainly a different candidate's own agent already — and once
3471    /// the roster's tail is exhausted there is nothing left to fall through
3472    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3473    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3474    /// let this retry that id forever. The loop keeps falling through until
3475    /// an attempt lands something other than `Quota` or the roster's tail
3476    /// runs out of untried ids, at which point the seat is left exactly as
3477    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3478    /// `QuotaLoss` recorded, the candidate failed/empty.
3479    ///
3480    /// `sent` is taken mutably and updated with the fallback agent's spec:
3481    /// `resume_unconfirmed_commands`, which runs after this and also reads
3482    /// `sent`, must see whichever agent actually ended up answering the seat
3483    /// — reading the stale, original spec there would check session
3484    /// eligibility against the wrong CLI and could hand a fallback agent's
3485    /// session id to the agent that just lost the seat to quota.
3486    ///
3487    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3488    /// — `self.seat` only reuses state when the agent id is unchanged, so
3489    /// handing it a different id already gets this for free. Reusing the old
3490    /// seat would resume a different CLI's session as if it were a
3491    /// continuation of this one.
3492    ///
3493    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3494    /// a quota loss cuts an agent off mid-turn, so anything already in the
3495    /// tree is unfinished work, not a completed candidate a re-ask would pay
3496    /// for twice. A dirty tree is rescued into a commit first (the same
3497    /// neutral-identity rescue `implement`'s own outcome loop gives every
3498    /// candidate) so the next agent starts clean.
3499    ///
3500    /// The new agent gets the implementer's full prompt and full
3501    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3502    /// it has no session and no context, and is implementing the task from
3503    /// nothing, unlike a resumed drop which is only restating work already
3504    /// done.
3505    ///
3506    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3507    /// `implement` event, never into `self.state.quota` — that is what
3508    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3509    /// go unspent, and a seat that ultimately recovered on its second or
3510    /// third agent is not the stalled panel that check exists to catch. Only
3511    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3512    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3513    /// outcome loop already has — this helper never pushes to it itself.
3514    async fn resume_seat_handovers(
3515        &mut self,
3516        results: &mut [(usize, SeatState, AgentOutcome)],
3517        sent: &mut [SeatJob],
3518        prompts: &Prompts,
3519        run_id: &str,
3520    ) -> Result<()> {
3521        let instruction = seeded_instruction(&self.state);
3522        let language = self.state.config.graph.language.clone();
3523        let github_language = self.state.config.graph.github_language.clone();
3524        let brief = self
3525            .state
3526            .advice
3527            .as_ref()
3528            .and_then(|a| a.synthesis.as_deref())
3529            .map(str::to_owned);
3530        let attachments = self.state.attachments.clone();
3531        for (wi, seat, out) in results.iter_mut() {
3532            // Who holds the other candidate seats of this wave right now
3533            // (earlier handovers already written back to `sent`).
3534            let others: BTreeSet<String> = sent
3535                .iter()
3536                .enumerate()
3537                .filter(|(j, _)| j != wi)
3538                .map(|(_, j)| j.spec.id.clone())
3539                .collect();
3540            let Some(job) = sent.get_mut(*wi) else {
3541                continue;
3542            };
3543            // Where the seat's own original agent sits in the roster — the
3544            // fallback walk starts just past here, never at the front, so a
3545            // later candidate slot's quota loss does not fall back onto an
3546            // earlier slot's own agent.
3547            let start = self
3548                .roles
3549                .implementer_roster
3550                .iter()
3551                .position(|s| s.id == job.spec.id)
3552                .unwrap_or(0);
3553            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3554            let mut fallback_attempt = 0usize;
3555            let mut prev: Option<FailClass> = None;
3556            while let Some(cur) = FailClass::of(&*out) {
3557                if !should_hand_over(prev.as_ref(), &cur) {
3558                    break;
3559                }
3560                let Some(next) =
3561                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3562                        .cloned()
3563                else {
3564                    break;
3565                };
3566                tried.insert(next.id.clone());
3567                fallback_attempt += 1;
3568
3569                let coordination = self.coordinate(&job.cwd).await?;
3570                if let Ok(r) = git::rescue_commit(
3571                    &job.cwd,
3572                    &format!(
3573                        "magi: candidate {} (uncommitted work before {} fallback)",
3574                        seat.key,
3575                        if cur == FailClass::Quota {
3576                            "quota"
3577                        } else {
3578                            "handover"
3579                        }
3580                    ),
3581                )
3582                .await
3583                {
3584                    self.state.note_withheld("implement", &r.withheld);
3585                }
3586                drop(coordination);
3587
3588                record_handover(
3589                    &mut self.state,
3590                    "implement",
3591                    &seat.key,
3592                    &seat.agent,
3593                    &next.id,
3594                    &cur,
3595                    &fail_reason(&*out),
3596                );
3597                prev = Some(cur.clone());
3598
3599                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3600                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3601                // Kept in sync on `sent` itself, not just the local retry: a
3602                // later helper (`resume_unconfirmed_commands`) reads `sent`
3603                // after this one returns and must see whichever agent is now
3604                // occupying the seat, not the one that just quota'd out —
3605                // otherwise it would judge session/continuation eligibility
3606                // by the wrong CLI and could resend a fallback's session id
3607                // to the agent that lost it the seat in the first place.
3608                job.spec = next.clone();
3609                let mut retry = job.clone();
3610                retry.seat = new_seat;
3611                retry.prompt = prompt::implement_in(
3612                    &instruction,
3613                    &job.cwd.to_string_lossy(),
3614                    &language,
3615                    &github_language,
3616                    brief.as_deref(),
3617                    &attachments,
3618                );
3619                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3620                let cache = self.state.config.cache_dir();
3621                let ctx = WaveCtx {
3622                    carry_seats: false,
3623                    pauses: Some((&self.pause, &self.interrupt)),
3624                    run: run_id,
3625                    node: "implement",
3626                    prompts,
3627                    cache: cache.as_deref(),
3628                    round: None,
3629                };
3630                let (fallback_seat, fallback_out) = run_one(
3631                    retry,
3632                    Arc::clone(&self.sem),
3633                    &ctx,
3634                    &mut self.state,
3635                    fallback_attempt,
3636                )
3637                .await;
3638                *seat = fallback_seat;
3639                *out = fallback_out;
3640            }
3641        }
3642        Ok(())
3643    }
3644
3645    /// Ask an implement seat's own CLI to confirm what it started, once, when
3646    /// its reply reported a command whose completion status it never
3647    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3648    /// that does and does not mean.
3649    ///
3650    /// The completion contract this task asks for, extended to `implement`
3651    /// with the same signal `continue_fix_report` reads for the fixer,
3652    /// rather than a keyword search over the reply or a hard requirement on
3653    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3654    /// a candidate's CLI turn ended cleanly while a test run it had started
3655    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3656    /// named in it at all is untouched by this: `commands` is empty, so
3657    /// there is nothing to be unconfirmed.
3658    ///
3659    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3660    /// this is not about recovering edits that might already be on disk, it
3661    /// is about a result the seat itself never vouched for, which resuming
3662    /// asks for regardless of what the tree already holds. Bounded to one
3663    /// attempt for the same reason `resume_undelivered` is — this is the
3664    /// most expensive node in the graph — and a seat that still cannot
3665    /// confirm on that attempt is left as whatever its (possibly still
3666    /// unconfirmed) reply says; this does not invent a new "failed" reason
3667    /// for a candidate that otherwise produced a real, committed change.
3668    async fn resume_unconfirmed_commands(
3669        &mut self,
3670        results: &mut [(usize, SeatState, AgentOutcome)],
3671        sent: &[SeatJob],
3672        prompts: &Prompts,
3673        run_id: &str,
3674    ) {
3675        for (wi, seat, out) in results.iter_mut() {
3676            let AgentOutcome::Ok(o) = &*out else {
3677                continue;
3678            };
3679            if !has_unconfirmed_command(&o.commands) {
3680                continue;
3681            }
3682            let Some(job) = sent.get(*wi) else { continue };
3683            if !has_context(&job.spec, seat, job.sessions) {
3684                self.state.event(
3685                    "implement",
3686                    format!(
3687                        "{}: the reply named a command whose own CLI never confirmed the exit \
3688                         status of, but there is no session left to resume",
3689                        seat.key
3690                    ),
3691                );
3692                continue;
3693            }
3694            self.state.event(
3695                "implement",
3696                format!(
3697                    "{}: the reply named a command whose own CLI never confirmed the exit \
3698                     status of; resuming the conversation",
3699                    seat.key
3700                ),
3701            );
3702            let mut retry = job.clone();
3703            retry.seat = seat.clone();
3704            retry.prompt = prompt::resume_incomplete(
3705                "a command in your last reply had no confirmed exit status",
3706            );
3707            retry.timeout = retry_budget(job.timeout, true);
3708            retry.stem = format!("{}-confirm", job.stem);
3709            let cache = self.state.config.cache_dir();
3710            let ctx = WaveCtx {
3711                carry_seats: false,
3712                pauses: Some((&self.pause, &self.interrupt)),
3713                run: run_id,
3714                node: "implement",
3715                prompts,
3716                cache: cache.as_deref(),
3717                round: None,
3718            };
3719            let (resumed_seat, resumed) =
3720                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3721            *seat = resumed_seat;
3722            *out = resumed;
3723        }
3724    }
3725
3726    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3727    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3728    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3729    /// that motivated this.
3730    ///
3731    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3732    /// own nudge loop already covers for judge/review/vote seats, and not a
3733    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3734    /// implement seats: here the CLI turn genuinely finished while the node's
3735    /// own work — the fixer's account of what it did — had not. Gated purely
3736    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3737    /// reply, never on any wording in it, so a fixer whose valid, first-try
3738    /// `FixReport` happens to mention having waited on a background test is
3739    /// never resumed — the `Ok(report)` branch at the call site returns
3740    /// before this is ever invoked.
3741    ///
3742    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3743    /// attempt ([`retry_budget`]), nothing attempted once the session is
3744    /// gone, and a quota hit ends the loop immediately rather than retrying a
3745    /// rate limit that fails the same way again.
3746    async fn continue_fix_report(
3747        &mut self,
3748        mut seat: SeatState,
3749        parse_err: String,
3750        job: &SeatJob,
3751        prompts: &Prompts,
3752        run_id: &str,
3753        round: usize,
3754    ) -> (
3755        SeatState,
3756        Option<FixReport>,
3757        Option<String>,
3758        ContinuationRecord,
3759    ) {
3760        let mut last_err = parse_err;
3761        let mut cumulative_wait_ms = 0u64;
3762        let mut attempts = 0usize;
3763        loop {
3764            if !has_context(&job.spec, &seat, job.sessions) {
3765                self.state.event(
3766                    "fix",
3767                    format!(
3768                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3769                         session left to resume into"
3770                    ),
3771                );
3772                let outcome = if attempts == 0 {
3773                    ContinuationOutcome::NoSession
3774                } else {
3775                    ContinuationOutcome::Exhausted
3776                };
3777                return (
3778                    seat,
3779                    None,
3780                    Some(format!("unparsable fix report: {last_err}")),
3781                    ContinuationRecord {
3782                        attempts,
3783                        cumulative_wait_ms,
3784                        outcome,
3785                    },
3786                );
3787            }
3788            if attempts >= MAX_FIX_CONTINUATIONS {
3789                self.state.event(
3790                    "fix",
3791                    format!(
3792                        "round {round}: fixer's reply still had no adoption report after \
3793                         {attempts} continuation(s) ({last_err}); giving up"
3794                    ),
3795                );
3796                return (
3797                    seat,
3798                    None,
3799                    Some(format!(
3800                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3801                    )),
3802                    ContinuationRecord {
3803                        attempts,
3804                        cumulative_wait_ms,
3805                        outcome: ContinuationOutcome::Exhausted,
3806                    },
3807                );
3808            }
3809            attempts += 1;
3810            self.state.event(
3811                "fix",
3812                format!(
3813                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3814                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3815                ),
3816            );
3817            let mut retry = job.clone();
3818            retry.seat = seat.clone();
3819            retry.prompt = prompt::resume_incomplete(&last_err);
3820            retry.timeout = retry_budget(job.timeout, true);
3821            retry.stem = format!("{}-continue{attempts}", job.stem);
3822            let cache = self.state.config.cache_dir();
3823            let ctx = WaveCtx {
3824                carry_seats: false,
3825                pauses: Some((&self.pause, &self.interrupt)),
3826                run: run_id,
3827                node: "fix",
3828                prompts,
3829                cache: cache.as_deref(),
3830                round: Some(round),
3831            };
3832            let (resumed_seat, resumed_out) = run_one(
3833                retry,
3834                Arc::clone(&self.sem),
3835                &ctx,
3836                &mut self.state,
3837                attempts,
3838            )
3839            .await;
3840            seat = resumed_seat;
3841            match resumed_out {
3842                AgentOutcome::Ok(o) => {
3843                    cumulative_wait_ms += o.duration_ms;
3844                    match verdict::extract_json::<FixReport>(&o.text) {
3845                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3846                            self.state.event(
3847                                "fix",
3848                                format!(
3849                                    "round {round}: fixer's adoption report recovered after \
3850                                     {attempts} continuation(s)"
3851                                ),
3852                            );
3853                            return (
3854                                seat,
3855                                Some(report),
3856                                None,
3857                                ContinuationRecord {
3858                                    attempts,
3859                                    cumulative_wait_ms,
3860                                    outcome: ContinuationOutcome::Resumed,
3861                                },
3862                            );
3863                        }
3864                        // The report parsed, but this same reply's own
3865                        // CommandEvidence — the identical record `state.jobs`
3866                        // renders — names a command whose CLI never
3867                        // confirmed an exit status. Read together, that is
3868                        // not a resolved answer: keep nudging rather than
3869                        // accept a report standing next to a command the
3870                        // seat's own CLI cannot vouch for.
3871                        Ok(_) => {
3872                            last_err = "the reply parsed, but it reported a command whose own CLI \
3873                                 never confirmed an exit status"
3874                                .to_owned();
3875                        }
3876                        Err(e) => last_err = e.to_string(),
3877                    }
3878                }
3879                AgentOutcome::Quota(o) => {
3880                    cumulative_wait_ms += o.duration_ms;
3881                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3882                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3883                    self.state.quota.push(loss);
3884                    self.state.event(
3885                        "fix",
3886                        format!(
3887                            "round {round}: continuation rate limited (quota); not retrying now"
3888                        ),
3889                    );
3890                    return (
3891                        seat,
3892                        None,
3893                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3894                        ContinuationRecord {
3895                            attempts,
3896                            cumulative_wait_ms,
3897                            outcome: ContinuationOutcome::QuotaLost,
3898                        },
3899                    );
3900                }
3901                AgentOutcome::Dropped(o) => {
3902                    cumulative_wait_ms += o.duration_ms;
3903                    let why = o
3904                        .dropped
3905                        .as_ref()
3906                        .map(|d| d.why.as_str())
3907                        .unwrap_or("the CLI ended the stream without delivering its answer");
3908                    last_err = format!("the CLI dropped the stream ({why})");
3909                }
3910                AgentOutcome::Failed(e) => last_err = e,
3911                // The continuation was never started; the report stays lost
3912                // and the caller records that, without any handover.
3913                AgentOutcome::ResourceBlocked(e) => {
3914                    return (
3915                        seat,
3916                        None,
3917                        Some(format!("the continuation was not started: {e}")),
3918                        ContinuationRecord {
3919                            attempts: attempts.saturating_sub(1),
3920                            cumulative_wait_ms,
3921                            outcome: ContinuationOutcome::NoSession,
3922                        },
3923                    );
3924                }
3925            }
3926        }
3927    }
3928
3929    fn after_implement(&mut self) -> Result<()> {
3930        if !self.state.candidate_preservation.is_empty() {
3931            bail!("candidate preservation incomplete; judging is unsafe");
3932        }
3933        // Scan every candidate patch once the set is complete.
3934        if self.state.leaks.is_empty() {
3935            let cfg = self.state.config.blind.clone();
3936            let mut leaks = Vec::new();
3937            for c in &self.state.candidates {
3938                let Some(patch) =
3939                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3940                else {
3941                    continue;
3942                };
3943                leaks.extend(blind::scan(
3944                    &format!("candidate {} patch", c.label),
3945                    &patch,
3946                    &cfg.vendor_tokens,
3947                ));
3948            }
3949            if !leaks.is_empty() {
3950                let summary = leaks
3951                    .iter()
3952                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3953                    .collect::<Vec<_>>()
3954                    .join(", ");
3955                match cfg.on_leak {
3956                    LeakPolicy::Fail => {
3957                        self.state.status = RunStatus::Failed;
3958                        self.state
3959                            .event("blind", format!("vendor text in a patch: {summary}"));
3960                        self.state.leaks = leaks;
3961                        self.state.save()?;
3962                        self.settle_questions();
3963                        bail!(
3964                            "blind.on_leak = \"fail\" and vendor text reached a \
3965                             judged patch: {summary}"
3966                        );
3967                    }
3968                    LeakPolicy::Redact => self.state.event(
3969                        "blind",
3970                        format!("redacting vendor text for judging: {summary}"),
3971                    ),
3972                    LeakPolicy::Warn => self.state.event(
3973                        "blind",
3974                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3975                    ),
3976                }
3977                self.state.leaks = leaks;
3978            }
3979        }
3980
3981        if self.state.viable().is_empty() {
3982            if self.state.all_candidates_verified_noop() {
3983                // Every candidate agreed, with evidence the adoption guard
3984                // accepted, that nothing belongs in this worktree. That is
3985                // not the same fact as a candidate that simply failed to
3986                // write anything, and settling it as an ordinary `Failed`
3987                // (see `SCHEMA`'s doc for schema 10) is what let two of
3988                // task 391f's attempts burn a retry each re-discovering the
3989                // same already-landed fix. Terminal either way, so `judge`
3990                // must never run over an empty candidate set — unlike the
3991                // `Failed` branch below this returns `Ok`, not an error:
3992                // nothing here failed.
3993                self.state.status = RunStatus::VerifiedNoop;
3994                self.state.save()?;
3995                self.settle_questions();
3996                return Ok(());
3997            }
3998            self.state.status = RunStatus::Failed;
3999            self.state.save()?;
4000            self.settle_questions();
4001            bail!("no candidate produced a change; nothing to judge");
4002        }
4003        self.state.status = RunStatus::Judging;
4004        self.state.save()?;
4005        Ok(())
4006    }
4007
4008    // --------------------------------------------------------------- judge
4009
4010    async fn judge(&mut self) -> Result<()> {
4011        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4012        // agent files with `magi task add` name the run that paid for it. The
4013        // prompt overlay is cloned alongside it because the waves borrow it
4014        // while `self` is mutably borrowed by the node's own bookkeeping.
4015        let run_id = self.state.id.clone();
4016        let prompts = self.state.config.prompts.clone();
4017        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4018            return Ok(());
4019        }
4020        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4021        if viable.len() == 1 {
4022            // Recorded so this is a one-time event: `judgements` stays empty
4023            // either way, which without this flag is indistinguishable from
4024            // "not yet judged" on the next reentry — and status is left
4025            // untouched, so a later node's conclusion (e.g. `Blocked` after
4026            // the review budget ran out) survives a resume instead of being
4027            // clobbered back to `Judging` by this node running again.
4028            self.state.judge_skipped = true;
4029            self.state.event(
4030                "judge",
4031                format!(
4032                    "only candidate {} produced a change; judging skipped",
4033                    viable[0].label
4034                ),
4035            );
4036            self.state.save()?;
4037            return Ok(());
4038        }
4039        self.state.status = RunStatus::Judging;
4040
4041        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4042        let language = self.state.config.graph.language.clone();
4043        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4044        let sessions = self.state.config.graph.sessions;
4045        let artifacts = agent::artifacts_dir(&self.state.dir());
4046        let root = self.state.worktree_root();
4047        let base_short = short(&self.state.base_commit);
4048
4049        let mut jobs = Vec::new();
4050        let mut orders = Vec::new();
4051        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4052            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4053            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4054            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4055            let seat_key = format!("judge-{}", j + 1);
4056            let seat = self.seat(&seat_key, &spec.id);
4057            jobs.push(SeatJob {
4058                prompt: prompt::judge(
4059                    &self.state.instruction,
4060                    &views,
4061                    self.roles.judges.len(),
4062                    &base_short,
4063                    &language,
4064                ),
4065                spec,
4066                seat,
4067                cwd: root.join(format!("judge-{}", j + 1)),
4068                timeout,
4069                allow_write: false,
4070                sessions,
4071                artifacts: artifacts.clone(),
4072                stem: format!("judge-{}", j + 1),
4073                handover: None,
4074            });
4075        }
4076
4077        self.state.event(
4078            "judge",
4079            format!(
4080                "{} judges ranking {} candidates blind",
4081                jobs.len(),
4082                viable.len()
4083            ),
4084        );
4085        let labels_for_check = labels.clone();
4086        let mut quota_losses = Vec::new();
4087        let cache = self.state.config.cache_dir();
4088        let ctx = WaveCtx {
4089            carry_seats: false,
4090            pauses: Some((&self.pause, &self.interrupt)),
4091            run: &run_id,
4092            node: "judge",
4093            prompts: &prompts,
4094            cache: cache.as_deref(),
4095            round: None,
4096        };
4097        let results = ask_json_wave::<Ranking>(
4098            jobs,
4099            Arc::clone(&self.sem),
4100            self.state.config.graph.retries,
4101            &self.roles.judge_roster,
4102            &ctx,
4103            &mut quota_losses,
4104            &mut self.state,
4105            &move |r: &Ranking| r.validate(&labels_for_check),
4106        )
4107        .await;
4108        if self.state.resource_block.is_some() {
4109            return self.stop_for_resource();
4110        }
4111        self.state.quota.extend(quota_losses);
4112
4113        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4114            let agent_id = seat.agent.clone();
4115            self.state.seats.insert(seat.key.clone(), seat);
4116            let mut record = Judgement {
4117                judge: j + 1,
4118                seat: format!("judge-{}", j + 1),
4119                agent: agent_id,
4120                ranking: Vec::new(),
4121                reasons: BTreeMap::new(),
4122                confidence: None,
4123                strengths: BTreeMap::new(),
4124                order: orders[j].clone(),
4125                failed: None,
4126                duration_ms: 0,
4127            };
4128            match res {
4129                Ok((ranking, out)) => {
4130                    record.ranking = ranking.normalized();
4131                    record.reasons = ranking.reasons;
4132                    record.strengths = ranking.strengths;
4133                    record.confidence = ranking.confidence;
4134                    record.duration_ms = out.duration_ms;
4135                    self.state.event(
4136                        "judge",
4137                        format!(
4138                            "judge {} ranked {}",
4139                            j + 1,
4140                            record.ranking.iter().collect::<String>()
4141                        ),
4142                    );
4143                }
4144                Err(e) => {
4145                    record.failed = Some(e.to_string());
4146                    self.state
4147                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4148                }
4149            }
4150            self.state.judgements.push(record);
4151            self.state.save()?;
4152        }
4153        Ok(())
4154    }
4155
4156    // ---------------------------------------------------------- deliberate
4157
4158    async fn deliberate(&mut self) -> Result<()> {
4159        if self.hold_base_sync().await? {
4160            return Ok(());
4161        }
4162        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4163        // agent files with `magi task add` name the run that paid for it. The
4164        // prompt overlay is cloned alongside it because the waves borrow it
4165        // while `self` is mutably borrowed by the node's own bookkeeping.
4166        let run_id = self.state.id.clone();
4167        let prompts = self.state.config.prompts.clone();
4168        let rounds = self.state.config.graph.deliberate_rounds;
4169        // Resume unfinished rounds after an outage, but never reopen a tally.
4170        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4171            return Ok(());
4172        }
4173        let tops: Vec<char> = self
4174            .state
4175            .judgements
4176            .iter()
4177            .filter_map(|j| j.ranking.first().copied())
4178            .collect();
4179        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4180            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4181                self.state.event(
4182                    "deliberate",
4183                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4184                );
4185            }
4186            self.state.status = RunStatus::Voting;
4187            self.state.save()?;
4188            return Ok(());
4189        }
4190
4191        self.state.status = RunStatus::Deliberating;
4192        self.state.event(
4193            "deliberate",
4194            format!(
4195                "split: first choices were {} — opening {rounds} round(s)",
4196                tops.iter().collect::<String>()
4197            ),
4198        );
4199
4200        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4201        let language = self.state.config.graph.language.clone();
4202        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4203        let sessions = self.state.config.graph.sessions;
4204        let artifacts = agent::artifacts_dir(&self.state.dir());
4205        let root = self.state.worktree_root();
4206        let base_short = short(&self.state.base_commit);
4207
4208        // Judges argue in sequence so that a turn can answer the one before it;
4209        // that is the difference between deliberation and three parallel
4210        // monologues.
4211        for round in self.state.deliberation.len() + 1..=rounds {
4212            let mut turns: Vec<DeliberationTurn> = Vec::new();
4213            let mut round_losses = Vec::new();
4214            let mut asked = 0;
4215            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4216                if self.state.judgements[j].failed.is_some() {
4217                    continue;
4218                }
4219                let seat_key = format!("judge-{}", j + 1);
4220                let spec = self.occupant(&seat_key, spec);
4221                let mut seat = self.seat(&seat_key, &spec.id);
4222                let transcript = self.transcript(&turns, j);
4223                let build = |context: Option<&str>| {
4224                    prompt::deliberate(
4225                        &self.state.instruction,
4226                        context,
4227                        &transcript,
4228                        round,
4229                        rounds,
4230                        &language,
4231                    )
4232                };
4233                let block = self.candidate_block(&viable, &base_short);
4234                let full = build(Some(&block));
4235                let text = if has_context(&spec, &seat, sessions) {
4236                    build(None)
4237                } else {
4238                    full.clone()
4239                };
4240                let job = SeatJob {
4241                    spec,
4242                    seat: seat.clone(),
4243                    prompt: text,
4244                    cwd: root.join(format!("judge-{}", j + 1)),
4245                    timeout,
4246                    allow_write: false,
4247                    sessions,
4248                    artifacts: artifacts.clone(),
4249                    stem: format!("delib-{round}-judge-{}", j + 1),
4250                    handover: Some(full),
4251                };
4252                let cache = self.state.config.cache_dir();
4253                let ctx = WaveCtx {
4254                    carry_seats: false,
4255                    pauses: Some((&self.pause, &self.interrupt)),
4256                    run: &run_id,
4257                    node: "deliberate",
4258                    prompts: &prompts,
4259                    cache: cache.as_deref(),
4260                    round: None,
4261                };
4262                // A turn is never nudged (`retries` 0): a failed seat is
4263                // handed to the next roster agent, which gets the full
4264                // context. An empty answer is a turn, not a failure.
4265                let mut losses = Vec::new();
4266                asked += 1;
4267                let mut results = ask_wave_with::<String>(
4268                    vec![job],
4269                    Arc::clone(&self.sem),
4270                    0,
4271                    &self.roles.judge_roster,
4272                    &ctx,
4273                    &mut losses,
4274                    &mut self.state,
4275                    &|text: &str| {
4276                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4277                    },
4278                )
4279                .await;
4280                if self.state.resource_block.is_some() {
4281                    return self.stop_for_resource();
4282                }
4283                round_losses.extend(losses.iter().cloned());
4284                self.state.quota.extend(losses);
4285                let (updated, res, _) = results.pop().expect("one job in, one result out");
4286                seat = updated;
4287                let agent_id = seat.agent.clone();
4288                self.state.seats.insert(seat.key.clone(), seat);
4289                let body = match res {
4290                    Ok((body, _)) => body,
4291                    // Skip the seat; a CLI's raw error JSON is never read as
4292                    // this judge's position.
4293                    Err(e) => {
4294                        self.state
4295                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4296                        continue;
4297                    }
4298                };
4299                let tentative = verdict::extract_json::<Position>(&body)
4300                    .ok()
4301                    .and_then(|p| p.tentative)
4302                    .and_then(|s| s.trim().chars().next())
4303                    .map(|c| c.to_ascii_uppercase());
4304                self.state.event(
4305                    "deliberate",
4306                    format!(
4307                        "round {round}: judge {} now favours {}",
4308                        j + 1,
4309                        tentative.map_or("—".to_owned(), |c| c.to_string())
4310                    ),
4311                );
4312                turns.push(DeliberationTurn {
4313                    judge: j + 1,
4314                    agent: agent_id,
4315                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4316                    tentative,
4317                });
4318            }
4319            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4320                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4321                return self.stop_for_resource();
4322            }
4323            self.state
4324                .deliberation
4325                .push(DeliberationRound { round, turns });
4326            self.state.save()?;
4327        }
4328
4329        self.state.status = RunStatus::Voting;
4330        self.state.save()?;
4331        Ok(())
4332    }
4333
4334    // ---------------------------------------------------------------- vote
4335
4336    async fn vote(&mut self) -> Result<()> {
4337        if self.hold_base_sync().await? {
4338            return Ok(());
4339        }
4340        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4341        // agent files with `magi task add` name the run that paid for it. The
4342        // prompt overlay is cloned alongside it because the waves borrow it
4343        // while `self` is mutably borrowed by the node's own bookkeeping.
4344        let run_id = self.state.id.clone();
4345        let prompts = self.state.config.prompts.clone();
4346        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4347            return Ok(());
4348        }
4349        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4350        if viable.len() == 1 {
4351            return Ok(());
4352        }
4353        self.state.status = RunStatus::Voting;
4354
4355        let language = self.state.config.graph.language.clone();
4356        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4357        let sessions = self.state.config.graph.sessions;
4358        let artifacts = agent::artifacts_dir(&self.state.dir());
4359        let root = self.state.worktree_root();
4360        let base_short = short(&self.state.base_commit);
4361        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4362
4363        let mut jobs = Vec::new();
4364        let mut seats_at = Vec::new();
4365        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4366            if self
4367                .state
4368                .judgements
4369                .get(j)
4370                .is_some_and(|r| r.failed.is_some())
4371            {
4372                continue;
4373            }
4374            let seat_key = format!("judge-{}", j + 1);
4375            let spec = self.occupant(&seat_key, spec);
4376            let seat = self.seat(&seat_key, &spec.id);
4377            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4378            let text = if has_context(&spec, &seat, sessions) {
4379                prompt::final_vote(&viable, &language)
4380            } else {
4381                full.clone()
4382            };
4383            jobs.push(SeatJob {
4384                spec,
4385                seat,
4386                prompt: text,
4387                cwd: root.join(format!("judge-{}", j + 1)),
4388                timeout,
4389                allow_write: false,
4390                sessions,
4391                artifacts: artifacts.clone(),
4392                stem: format!("vote-judge-{}", j + 1),
4393                handover: Some(full),
4394            });
4395            seats_at.push(j);
4396        }
4397
4398        self.state.event(
4399            "vote",
4400            format!(
4401                "collecting {} final votes one by one, privately",
4402                jobs.len()
4403            ),
4404        );
4405        let allowed = viable.clone();
4406        let mut quota_losses = Vec::new();
4407        let cache = self.state.config.cache_dir();
4408        let ctx = WaveCtx {
4409            carry_seats: false,
4410            pauses: Some((&self.pause, &self.interrupt)),
4411            run: &run_id,
4412            node: "vote",
4413            prompts: &prompts,
4414            cache: cache.as_deref(),
4415            round: None,
4416        };
4417        let results = ask_json_wave::<FinalVote>(
4418            jobs,
4419            Arc::clone(&self.sem),
4420            self.state.config.graph.retries,
4421            &self.roles.judge_roster,
4422            &ctx,
4423            &mut quota_losses,
4424            &mut self.state,
4425            &move |v: &FinalVote| match v.label() {
4426                Some(c) if allowed.contains(&c) => Ok(()),
4427                other => bail!("vote {other:?} is not one of {allowed:?}"),
4428            },
4429        )
4430        .await;
4431        if self.state.resource_block.is_some() {
4432            return self.stop_for_resource();
4433        }
4434        self.state.quota.extend(quota_losses);
4435
4436        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4437            let agent_id = seat.agent.clone();
4438            self.state.seats.insert(seat.key.clone(), seat);
4439            let initial = self
4440                .state
4441                .judgements
4442                .get(j)
4443                .and_then(|r| r.ranking.first().copied());
4444            let mut record = VoteRecord {
4445                judge: j + 1,
4446                agent: agent_id,
4447                vote: None,
4448                reason: String::new(),
4449                changed: false,
4450            };
4451            match res {
4452                Ok((v, _)) => {
4453                    record.vote = v.label();
4454                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4455                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4456                    self.state.event(
4457                        "vote",
4458                        format!(
4459                            "judge {} voted {}{}",
4460                            j + 1,
4461                            record.vote.unwrap_or('?'),
4462                            if record.changed { " (changed)" } else { "" }
4463                        ),
4464                    );
4465                }
4466                Err(e) => {
4467                    self.state
4468                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4469                }
4470            }
4471            self.state.votes.push(record);
4472            self.state.save()?;
4473        }
4474        Ok(())
4475    }
4476
4477    // --------------------------------------------------------------- tally
4478
4479    fn tally(&mut self) -> Result<()> {
4480        if self.state.tally.is_some() {
4481            return Ok(());
4482        }
4483        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4484        let tops: Vec<char> = self
4485            .state
4486            .judgements
4487            .iter()
4488            .filter_map(|j| j.ranking.first().copied())
4489            .collect();
4490        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4491
4492        // A judge whose private vote failed still counted once, in the initial
4493        // ranking; using it beats discarding a whole seat.
4494        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4495        let mut cast: Vec<char> = Vec::new();
4496        for (i, j) in self.state.judgements.iter().enumerate() {
4497            let vote = self
4498                .state
4499                .votes
4500                .iter()
4501                .find(|v| v.judge == i + 1)
4502                .and_then(|v| v.vote)
4503                .or_else(|| j.ranking.first().copied());
4504            if let Some(v) = vote {
4505                *first_choice.entry(v).or_insert(0) += 1;
4506                cast.push(v);
4507            }
4508        }
4509
4510        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4511        for j in &self.state.judgements {
4512            let n = j.ranking.len();
4513            for (pos, label) in j.ranking.iter().enumerate() {
4514                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4515            }
4516        }
4517
4518        let best = first_choice.values().copied().max().unwrap_or(0);
4519        let mut leaders: Vec<char> = first_choice
4520            .iter()
4521            .filter(|(_, v)| **v == best)
4522            .map(|(k, _)| *k)
4523            .collect();
4524        let mut tie_break = None;
4525        if leaders.len() > 1 {
4526            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4527            let borda_leaders: Vec<char> = leaders
4528                .iter()
4529                .copied()
4530                .filter(|l| borda[l] == top_borda)
4531                .collect();
4532            tie_break = Some(if borda_leaders.len() == 1 {
4533                format!(
4534                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4535                    leaders.len()
4536                )
4537            } else {
4538                format!(
4539                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4540                    leaders.len()
4541                )
4542            });
4543            leaders = borda_leaders;
4544            leaders.sort_unstable();
4545        }
4546        let winner = *leaders
4547            .first()
4548            .or(viable.first())
4549            .context("no candidate to declare a winner from")?;
4550
4551        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4552        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4553        let deliberated = !self.state.deliberation.is_empty();
4554
4555        // Whose verdict is this? A rate-limited seat is absent even if it
4556        // ranked before the limit hit, so presence is measured against the
4557        // recorded losses, not just "did a ranking ever appear".
4558        let quota_seats: std::collections::BTreeSet<&str> =
4559            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4560        let mut present = 0usize;
4561        for (i, j) in self.state.judgements.iter().enumerate() {
4562            if quota_seats.contains(j.seat.as_str()) {
4563                continue;
4564            }
4565            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4566            let voted = self
4567                .state
4568                .votes
4569                .iter()
4570                .any(|v| v.judge == i + 1 && v.vote.is_some());
4571            if ranked || voted {
4572                present += 1;
4573            }
4574        }
4575        // Strict majority of the configured panel. A bare majority is real
4576        // signal we can act on, while a minority verdict must never stand in
4577        // for a healthy one. A one-candidate run needs no panel at all, and
4578        // `judges` stays `0` rather than the roster size a panel that never
4579        // sat would otherwise be credited with.
4580        let needs_quorum = viable.len() > 1;
4581        let judges_total = if needs_quorum {
4582            self.roles.judges.len()
4583        } else {
4584            0
4585        };
4586        let quorum = if needs_quorum {
4587            judges_total / 2 + 1
4588        } else {
4589            0
4590        };
4591        let met_quorum = !needs_quorum || present >= quorum;
4592        let uncontested = (!needs_quorum).then(|| {
4593            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4594        });
4595
4596        self.state.event(
4597            "tally",
4598            match &uncontested {
4599                Some(reason) => format!("winner {winner} — {reason}"),
4600                None => format!(
4601                    "winner {winner} — votes {} | initial {} | {} changed | \
4602                     {present}/{judges_total} judges{}",
4603                    first_choice
4604                        .iter()
4605                        .map(|(k, v)| format!("{k}:{v}"))
4606                        .collect::<Vec<_>>()
4607                        .join(" "),
4608                    if unanimous_initial {
4609                        "unanimous"
4610                    } else {
4611                        "split"
4612                    },
4613                    changed_votes,
4614                    if met_quorum {
4615                        String::new()
4616                    } else {
4617                        format!(" — below quorum ({quorum} required)")
4618                    },
4619                ),
4620            },
4621        );
4622        if !met_quorum {
4623            self.state.event(
4624                "stall",
4625                format!(
4626                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4627                     the run stops here, resumable"
4628                ),
4629            );
4630        }
4631        self.state.tally = Some(Tally {
4632            first_choice,
4633            borda,
4634            winner,
4635            rankings: tops.len(),
4636            unanimous_initial,
4637            deliberated,
4638            changed_votes,
4639            unanimous_final,
4640            tie_break,
4641            judges: judges_total,
4642            present,
4643            quorum,
4644            met_quorum,
4645            uncontested,
4646        });
4647        self.state.status = if met_quorum {
4648            RunStatus::Reviewing
4649        } else {
4650            RunStatus::Stalled
4651        };
4652        self.state.save()?;
4653        Ok(())
4654    }
4655
4656    // ------------------------------------------------------------- recover
4657
4658    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4659    /// resumed toward completion once the transient cause clears.
4660    ///
4661    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4662    /// it toward the quorum, which is exactly the set of seats whose absence
4663    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4664    /// not depend on which node happened to hit the limit), or an ordinary
4665    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4666    /// seat is never disturbed.
4667    ///
4668    /// A seat that now answers with a usable ranking is "recovered": its
4669    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4670    /// `tally` counts it present again), and its vote re-collected. A seat that
4671    /// still fails keeps its loss and stays absent.
4672    ///
4673    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4674    /// to review/gate/merge), `false` when it is still below quorum (the run
4675    /// stays `Stalled`, still resumable for a later retry).
4676    #[allow(clippy::too_many_lines)]
4677    async fn recover_stall(&mut self) -> Result<bool> {
4678        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4679        // agent files with `magi task add` name the run that paid for it. The
4680        // prompt overlay is cloned alongside it because the waves borrow it
4681        // while `self` is mutably borrowed by the node's own bookkeeping.
4682        let run_id = self.state.id.clone();
4683        let prompts = self.state.config.prompts.clone();
4684        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4685        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4686        // is what keeps a non-quota collapse — or a quota loss recorded at the
4687        // deliberate node — from being a permanent dead-end on `--resume`.
4688        let quota_seats: BTreeSet<&str> =
4689            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4690        let absent: Vec<String> = self
4691            .state
4692            .judgements
4693            .iter()
4694            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4695            .map(|j| j.seat.clone())
4696            .collect();
4697        if absent.is_empty() {
4698            return Ok(false);
4699        }
4700        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4701        if viable.len() <= 1 {
4702            return Ok(false);
4703        }
4704        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4705        let language = self.state.config.graph.language.clone();
4706        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4707        let sessions = self.state.config.graph.sessions;
4708        let artifacts = agent::artifacts_dir(&self.state.dir());
4709        let root = self.state.worktree_root();
4710        let base_short = short(&self.state.base_commit);
4711        let candidates: Vec<Candidate> = viable.clone();
4712
4713        // Map each absent seat key to its 0-based position in `roles.judges`.
4714        let mut positions: Vec<usize> = absent
4715            .iter()
4716            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4717            .collect();
4718        if positions.is_empty() {
4719            return Ok(false);
4720        }
4721        positions.sort_unstable();
4722        positions.dedup();
4723
4724        // Re-rank the lost seats, one blind prompt each.
4725        let mut judge_jobs = Vec::new();
4726        for &j in &positions {
4727            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4728            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4729            let seat_key = format!("judge-{}", j + 1);
4730            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4731            let seat = self.seat(&seat_key, &spec.id);
4732            judge_jobs.push(SeatJob {
4733                spec,
4734                seat,
4735                prompt: prompt::judge(
4736                    &self.state.instruction,
4737                    &views,
4738                    self.roles.judges.len(),
4739                    &base_short,
4740                    &language,
4741                ),
4742                cwd: root.join(seat_key),
4743                timeout,
4744                allow_write: false,
4745                sessions,
4746                artifacts: artifacts.clone(),
4747                stem: format!("judge-{}-recover", j + 1),
4748                handover: None,
4749            });
4750        }
4751
4752        let labels_for_check = labels.clone();
4753        let mut judge_losses = Vec::new();
4754        let retries = self.state.config.graph.retries;
4755        let cache = self.state.config.cache_dir();
4756        let ctx = WaveCtx {
4757            carry_seats: false,
4758            pauses: Some((&self.pause, &self.interrupt)),
4759            run: &run_id,
4760            node: "judge",
4761            prompts: &prompts,
4762            cache: cache.as_deref(),
4763            round: None,
4764        };
4765        let results = ask_json_wave::<Ranking>(
4766            judge_jobs,
4767            Arc::clone(&self.sem),
4768            retries,
4769            &self.roles.judge_roster,
4770            &ctx,
4771            &mut judge_losses,
4772            &mut self.state,
4773            &move |r: &Ranking| r.validate(&labels_for_check),
4774        )
4775        .await;
4776        if self.state.resource_block.is_some() {
4777            self.stop_for_resource()?;
4778            return Ok(false);
4779        }
4780
4781        // Refresh the judgement of every seat that ranked again.
4782        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4783        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4784            let agent_id = seat.agent.clone();
4785            self.state.seats.insert(seat.key.clone(), seat);
4786            let record = &mut self.state.judgements[j];
4787            match res {
4788                Ok((ranking, out)) => {
4789                    record.agent = agent_id;
4790                    record.ranking = ranking.normalized();
4791                    record.reasons = ranking.reasons;
4792                    record.strengths = ranking.strengths;
4793                    record.confidence = ranking.confidence;
4794                    record.failed = None;
4795                    record.duration_ms = out.duration_ms;
4796                    recovered.insert(j);
4797                    self.state.event(
4798                        "recover",
4799                        format!("judge {} ranked again after the limit", j + 1),
4800                    );
4801                }
4802                Err(e) => {
4803                    self.state
4804                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4805                }
4806            }
4807        }
4808
4809        // Re-ask the votes of the seats that recovered a ranking.
4810        let mut vote_jobs = Vec::new();
4811        let mut vote_pos: Vec<usize> = Vec::new();
4812        for &j in &recovered {
4813            let seat_key = format!("judge-{}", j + 1);
4814            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4815            let seat = self.seat(&seat_key, &spec.id);
4816            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4817            let text = if has_context(&spec, &seat, sessions) {
4818                prompt::final_vote(&labels, &language)
4819            } else {
4820                full.clone()
4821            };
4822            vote_jobs.push(SeatJob {
4823                spec,
4824                seat,
4825                prompt: text,
4826                cwd: root.join(seat_key),
4827                timeout,
4828                allow_write: false,
4829                sessions,
4830                artifacts: artifacts.clone(),
4831                stem: format!("vote-judge-{}-recover", j + 1),
4832                handover: Some(full),
4833            });
4834            vote_pos.push(j);
4835        }
4836        let allowed = labels.clone();
4837        let mut vote_losses = Vec::new();
4838        let vote_retries = self.state.config.graph.retries;
4839        let vote_cache = self.state.config.cache_dir();
4840        let ctx = WaveCtx {
4841            carry_seats: false,
4842            pauses: Some((&self.pause, &self.interrupt)),
4843            run: &run_id,
4844            node: "vote",
4845            prompts: &prompts,
4846            cache: vote_cache.as_deref(),
4847            round: None,
4848        };
4849        let votes = ask_json_wave::<FinalVote>(
4850            vote_jobs,
4851            Arc::clone(&self.sem),
4852            vote_retries,
4853            &self.roles.judge_roster,
4854            &ctx,
4855            &mut vote_losses,
4856            &mut self.state,
4857            &move |v: &FinalVote| match v.label() {
4858                Some(c) if allowed.contains(&c) => Ok(()),
4859                other => bail!("vote {other:?} is not one of {allowed:?}"),
4860            },
4861        )
4862        .await;
4863        if self.state.resource_block.is_some() {
4864            self.stop_for_resource()?;
4865            return Ok(false);
4866        }
4867        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4868            let agent_id = seat.agent.clone();
4869            self.state.seats.insert(seat.key.clone(), seat);
4870            match res {
4871                Ok((v, _)) => {
4872                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4873                        rec.vote = v.label();
4874                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4875                    } else {
4876                        self.state.votes.push(VoteRecord {
4877                            judge: j + 1,
4878                            agent: agent_id,
4879                            vote: v.label(),
4880                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4881                            changed: false,
4882                        });
4883                    }
4884                    self.state.event(
4885                        "recover",
4886                        format!("judge {} voted again after the limit", j + 1),
4887                    );
4888                }
4889                Err(e) => {
4890                    self.state
4891                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4892                }
4893            }
4894        }
4895
4896        // A seat that ranked again is present even if its re-vote failed —
4897        // `tally` falls back to the initial ranking's first choice — so clear
4898        // its quota loss. Seats that still fail keep theirs and stay absent.
4899        let recovered_keys: BTreeSet<String> = recovered
4900            .iter()
4901            .map(|&j| format!("judge-{}", j + 1))
4902            .collect();
4903        self.state
4904            .quota
4905            .retain(|q| !recovered_keys.contains(&q.seat));
4906        // A seat that hit the limit again is a fresh loss, not the old one:
4907        // replace the stale entry so the history stays one-per-seat and the
4908        // daemon can tell this attempt's loss from a previous session's.
4909        for loss in judge_losses.into_iter().chain(vote_losses) {
4910            if recovered_keys.contains(&loss.seat) {
4911                continue;
4912            }
4913            self.state.quota.retain(|q| q.seat != loss.seat);
4914            self.state.quota.push(loss);
4915        }
4916
4917        // Recompute the verdict from the refreshed panel.
4918        self.state.tally = None;
4919        self.tally()?;
4920        Ok(self
4921            .state
4922            .tally
4923            .as_ref()
4924            .map(|t| t.met_quorum)
4925            .unwrap_or(false))
4926    }
4927
4928    // ---------------------------------------------------------------- graft
4929
4930    /// Port the strengths the judges attributed to the losing candidates onto
4931    /// the winner, once, between the tally and the review loop.
4932    ///
4933    /// Best-effort by construction: it returns nothing and fails nothing. A
4934    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4935    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4936    /// the run carries on as if the phase were off. See [`graft`] for the
4937    /// decisions and `AGENTS.md` for the invariants.
4938    ///
4939    /// The one exception is a rollback that could not be confirmed: the
4940    /// rejected commit may still be the winner's tip, so the run is blocked
4941    /// (resumable; the record stays `Running`, which makes a resume retry the
4942    /// restore) instead of reviewing, gating and merging it.
4943    async fn graft(&mut self) -> Result<()> {
4944        let Err(e) = self.graft_inner().await else {
4945            return Ok(());
4946        };
4947        if e.is::<crate::repo_lock::ResourceBlocked>() {
4948            return Err(e);
4949        }
4950        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4951            self.state.status = RunStatus::Blocked;
4952            self.state.event("graft", format!("blocked: {e:#}"));
4953            self.state.save()?;
4954            return Err(e);
4955        }
4956        // An error after the `Running` record was saved may have left the
4957        // fixer's work in the tree: the record must not stay unsettled
4958        // (a later resume would reset past the review fixes), so confirm the
4959        // restore now, or block.
4960        if let Some(rec) = self.state.graft.clone()
4961            && !rec.outcome.settled()
4962        {
4963            if let Some(winner) = self.state.winner().cloned() {
4964                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4965                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4966                        return Err(re);
4967                    }
4968                    self.state.status = RunStatus::Blocked;
4969                    self.state.event("graft", format!("blocked: {re:#}"));
4970                    self.state.save()?;
4971                    return Err(re);
4972                }
4973            }
4974            if let Some(g) = self.state.graft.as_mut() {
4975                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4976            }
4977        }
4978        self.state.event("graft", format!("skipped: {e:#}"));
4979        if let Err(e) = self.state.save() {
4980            tracing::warn!("save after graft: {e:#}");
4981        }
4982        Ok(())
4983    }
4984
4985    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4986    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4987    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4988        let _coordination = self.coordinate(wt).await?;
4989        crate::recovery::restore_repair(wt, pre)
4990            .await
4991            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4992    }
4993
4994    /// A skip is a run event, recorded once however often the node is re-entered.
4995    fn graft_skipped(&mut self, why: &str) {
4996        let message = format!("skipped: {why}");
4997        if !self
4998            .state
4999            .events
5000            .iter()
5001            .any(|e| e.node == "graft" && e.message == message)
5002        {
5003            self.state.event("graft", message);
5004        }
5005    }
5006
5007    async fn graft_inner(&mut self) -> Result<()> {
5008        // A round that was started and never finished (crash, kill): the
5009        // fixer may have left anything in the tree, so put the winner back.
5010        if let Some(rec) = self.state.graft.clone()
5011            && !rec.outcome.settled()
5012        {
5013            // Reviews exist only if the graft ended without its record being
5014            // settled (a swallowed error): the tree has moved on since, and
5015            // a reset to `pre` would discard the review fixes.
5016            if self.state.reviews.is_empty()
5017                && let Some(winner) = self.state.winner().cloned()
5018            {
5019                self.graft_restore(&winner.worktree, &rec.pre).await?;
5020            }
5021            if rec.outcome == GraftOutcome::WaitingForRestore {
5022                self.state.graft = None;
5023                self.state.save()?;
5024            } else {
5025                if let Some(g) = self.state.graft.as_mut() {
5026                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5027                }
5028                self.state.event(
5029                    "graft",
5030                    "rolled back: the previous round was interrupted; the winner is untouched",
5031                );
5032                self.state.save()?;
5033                return Ok(());
5034            }
5035        }
5036        let plan = match graft::plan(&self.state) {
5037            Ok(plan) => plan,
5038            Err(skip) => {
5039                if skip != graft::Skip::AlreadyDone {
5040                    self.graft_skipped(skip.reason());
5041                    self.state.save()?;
5042                }
5043                return Ok(());
5044            }
5045        };
5046        let Some(winner) = self.state.winner().cloned() else {
5047            return Ok(());
5048        };
5049        let wt = winner.worktree.clone();
5050        if !git::is_clean(&wt).await? {
5051            self.graft_skipped("the winner's worktree has uncommitted changes");
5052            self.state.save()?;
5053            return Ok(());
5054        }
5055        let pre = git::rev_parse(&wt, "HEAD").await?;
5056        // Saved before the fixer is called: a stop from here on is repaired
5057        // by the branch above on the next entry.
5058        self.state.graft = Some(GraftRecord {
5059            pre: pre.clone(),
5060            post: None,
5061            lines: None,
5062            limit: None,
5063            agent: None,
5064            outcome: GraftOutcome::Running,
5065            requested: graft::requested(&plan),
5066            adoption: None,
5067            adoption_error: None,
5068        });
5069        self.state.event(
5070            "graft",
5071            format!(
5072                "porting strengths of candidate(s) {} onto {}",
5073                plan.strengths
5074                    .iter()
5075                    .map(|(l, _, _)| l.to_string())
5076                    .collect::<Vec<_>>()
5077                    .join(", "),
5078                plan.winner
5079            ),
5080        );
5081        self.state.save()?;
5082
5083        let fix_prompt = prompt::graft(
5084            &self.state.instruction,
5085            plan.winner,
5086            &plan.strengths,
5087            &self.state.config.graph.language,
5088        );
5089        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5090        let sessions = self.state.config.graph.sessions;
5091        let artifacts = agent::artifacts_dir(&self.state.dir());
5092        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5093        let (_, seat, out) = self
5094            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5095                SeatJob {
5096                    prompt: fix_prompt.clone(),
5097                    spec,
5098                    seat,
5099                    cwd: wt.clone(),
5100                    timeout,
5101                    allow_write: true,
5102                    sessions,
5103                    artifacts: artifacts.clone(),
5104                    stem: "graft".to_owned(),
5105                    handover: None,
5106                }
5107            })
5108            .await;
5109        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5110            // Preserve the need to restore earlier fallback edits without
5111            // confusing an unstarted seat with an interrupted invocation.
5112            self.state.graft.as_mut().expect("graft record").outcome =
5113                GraftOutcome::WaitingForRestore;
5114            return self.stop_for_resource();
5115        }
5116        let mut why: Option<String> = None;
5117        match &out {
5118            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5119                why = Some("the fixer produced nothing usable".to_owned());
5120            }
5121            AgentOutcome::Ok(_) => {}
5122            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5123            AgentOutcome::Quota(o) => {
5124                self.state.quota.push(QuotaLoss::from_report(
5125                    seat.key.clone(),
5126                    "graft",
5127                    o.quota.as_ref(),
5128                ));
5129                why = Some("rate limited (quota)".to_owned());
5130            }
5131            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5132                why = Some(format!(
5133                    "the fixer failed: {}",
5134                    e.lines()
5135                        .next()
5136                        .unwrap_or("")
5137                        .chars()
5138                        .take(160)
5139                        .collect::<String>()
5140                ));
5141            }
5142        }
5143        let agent_id = seat.agent.clone();
5144        self.state.seats.insert(seat.key.clone(), seat);
5145
5146        let mut post = None;
5147        let mut lines = None;
5148        let _coordination = self.coordinate(&wt).await?;
5149        if why.is_none() {
5150            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5151            if let Ok(r) = &rescued {
5152                self.state.note_withheld("graft", &r.withheld);
5153            }
5154            let head = git::rev_parse(&wt, "HEAD").await?;
5155            // What is measured must be what gets reviewed and gated: a failed
5156            // rescue, or anything still uncommitted (withheld lockfiles
5157            // aside, which are never part of the tree), rejects the graft.
5158            let dirty = match &rescued {
5159                Ok(r) => git::status_porcelain(&wt)
5160                    .await
5161                    .map(|st| {
5162                        st.lines().any(|l| {
5163                            !r.withheld
5164                                .iter()
5165                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5166                        })
5167                    })
5168                    .unwrap_or(true),
5169                Err(_) => true,
5170            };
5171            if dirty {
5172                why = Some("the fixer left changes that could not be committed".to_owned());
5173            } else if head == pre {
5174                why = Some("the fixer changed nothing".to_owned());
5175            } else {
5176                lines = git::changed_lines(&wt, &pre, &head).await;
5177                post = Some(head);
5178            }
5179        }
5180        let (adoption, adoption_error) = match &out {
5181            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5182            _ => (None, None),
5183        };
5184        let requested = graft::requested(&plan);
5185        let outcome = match why {
5186            None => {
5187                self.state.event(
5188                    "graft",
5189                    format!(
5190                        "applied: {}",
5191                        lines.map_or_else(
5192                            || "changed lines unavailable".to_owned(),
5193                            |n| format!("{n} changed lines")
5194                        )
5195                    ),
5196                );
5197                GraftOutcome::Applied
5198            }
5199            Some(why) => {
5200                // Not settled until the restore is confirmed: the record
5201                // stays `Running`, so a resume retries it.
5202                crate::recovery::restore_repair(&wt, &pre)
5203                    .await
5204                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5205                self.state.event(
5206                    "graft",
5207                    format!("rolled back: {why}; the winner is untouched"),
5208                );
5209                GraftOutcome::RolledBack(why)
5210            }
5211        };
5212        self.state.graft = Some(GraftRecord {
5213            pre,
5214            post,
5215            lines,
5216            limit: None,
5217            agent: Some(agent_id),
5218            outcome,
5219            requested,
5220            adoption,
5221            adoption_error,
5222        });
5223        self.state.save()?;
5224        Ok(())
5225    }
5226
5227    // ----------------------------------------------------------------- fold
5228
5229    async fn fold_losers(&mut self) -> Result<()> {
5230        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5231            return Ok(());
5232        };
5233        if !self.state.candidate_preservation.is_empty() {
5234            bail!("candidate preservation incomplete; refusing cleanup");
5235        }
5236        let repo = self.state.repo.clone();
5237        let _coordination = self.coordinate(&repo).await?;
5238        let mut folded = Vec::new();
5239        for i in 0..self.state.candidates.len() {
5240            let c = &self.state.candidates[i];
5241            if c.label == winner || c.folded {
5242                continue;
5243            }
5244            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5245            remove_preserved_worktree(&repo, &wt).await?;
5246            if git::branch_exists(&repo, &branch).await?
5247                && !git::branch_delete(&repo, &branch).await?
5248            {
5249                bail!("candidate branch removal refused");
5250            }
5251            self.state.candidates[i].folded = true;
5252            self.state.save()?;
5253            folded.push(label.to_string());
5254        }
5255        // The judges are finished; their checkouts are pure cost from here.
5256        let root = self.state.worktree_root();
5257        for j in 1..=self.roles.judges.len() {
5258            let wt = root.join(format!("judge-{j}"));
5259            if wt.exists() {
5260                remove_preserved_worktree(&repo, &wt).await?;
5261            }
5262        }
5263        // The design-deliberation stage is finished by the time a tally
5264        // exists — same reasoning as the judges above.
5265        if self.state.config.graph.advise {
5266            for k in 1..=self.state.config.graph.advisors {
5267                let wt = root.join(format!("advisor-{k}"));
5268                if wt.exists() {
5269                    remove_preserved_worktree(&repo, &wt).await?;
5270                }
5271            }
5272        }
5273        if !folded.is_empty() {
5274            self.state
5275                .event("fold", format!("folded candidates {}", folded.join(", ")));
5276            self.state.save()?;
5277        }
5278        Ok(())
5279    }
5280
5281    // ------------------------------------------------------------ base sync
5282
5283    async fn hold_base_sync(&mut self) -> Result<bool> {
5284        self.hold_base_sync_as(None).await
5285    }
5286
5287    async fn hold_base_sync_kind(
5288        &mut self,
5289        kind: crate::base_sync_recovery::StopKind,
5290    ) -> Result<()> {
5291        self.hold_base_sync_as(Some(kind)).await?;
5292        Ok(())
5293    }
5294
5295    async fn hold_base_sync_as(
5296        &mut self,
5297        kind: Option<crate::base_sync_recovery::StopKind>,
5298    ) -> Result<bool> {
5299        if !self.state.base_sync_blocked() {
5300            self.state.base_sync_recovery.stop = None;
5301            return Ok(false);
5302        }
5303        let head = match self.state.winner() {
5304            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5305                .await
5306                .unwrap_or_default(),
5307            None => String::new(),
5308        };
5309        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5310            && !authorization.consumed
5311            && authorization.stop.head != head
5312        {
5313            authorization.refuse(format!(
5314                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5315                authorization.stop.head
5316            ));
5317        }
5318        self.state.record_base_sync_stop(&head);
5319        if let Some(kind) = kind {
5320            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5321            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5322                stop.kind = kind;
5323            }
5324        }
5325        self.state.save()?;
5326        Ok(true)
5327    }
5328
5329    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5330    /// anything verifies it.
5331    ///
5332    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5333    /// read whatever is checked out in the winner's worktree. Left alone that
5334    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5335    /// it when the run *branched* - and a run takes long enough that the base
5336    /// has usually moved by the time it gets here. A gate that ran there
5337    /// answers "green on the commit this run started from", not "green on
5338    /// what is about to land", and the difference showed up three times in
5339    /// one day as a green run whose merge would have reverted a file another
5340    /// pull request had already landed.
5341    ///
5342    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5343    /// `land::Step::Rebase` calls, rather than a second implementation of the
5344    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5345    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5346    /// doc warns about ("two answers to a question nobody notices until a
5347    /// diff is wrong").
5348    ///
5349    /// Foreign-checkout reviews instead merge the base into the owned shadow
5350    /// worktree, preserving the ancestry required for fast-forward delivery.
5351    /// A conflicting merge is aborted and held for operator resolution.
5352    ///
5353    /// A conflict is not the end of the road: the standing rebase is handed
5354    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5355    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5356    /// land). Once it finishes, review and the gate run as usual on the
5357    /// rebased tree, which is where a breakage the new base caused is caught
5358    /// by the ordinary gate-fix round. magi resolves nothing itself.
5359    ///
5360    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5361    /// the current durable recovery interval, bounds how often the base is
5362    /// synchronized during active recovery (lifetime attempts remain evidence).
5363    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5364    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5365    /// what was tried (rounds spent, paths still conflicted) and the branch
5366    /// and worktree stay exactly as they were - untouched, for a person to
5367    /// look at - which is also what makes re-entering this function
5368    /// afterwards a no-op instead of a second attempt at the same wall. A
5369    /// push failure ends the same way.
5370    async fn sync_to_base(&mut self) -> Result<()> {
5371        if self.state.status == RunStatus::AlreadyInBase {
5372            return Ok(());
5373        }
5374        self.hold_base_sync().await?;
5375        let conflicted = self
5376            .state
5377            .base_sync
5378            .as_ref()
5379            .is_some_and(|s| s.conflict.is_some());
5380        let Some(winner) = self.state.winner().cloned() else {
5381            return Ok(());
5382        };
5383
5384        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5385
5386        let repo = self.state.repo.clone();
5387        let remote = self.state.config.merge.remote.clone();
5388        let base_branch = self.state.base_branch.clone();
5389        let tracking = format!("{remote}/{base_branch}");
5390
5391        let fetched_base =
5392            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5393        // No network, or the remote never had this branch: the run already
5394        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5395        // otherwise), and one that got this far is not blocked by a fetch
5396        // that fails now. It just has no newer tip to compare against.
5397        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5398            return Ok(());
5399        };
5400
5401        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5402        self.state.ensure_base_episode();
5403        let mut pending = self
5404            .state
5405            .base_sync_recovery
5406            .operations
5407            .last()
5408            .filter(|o| !o.finished)
5409            .cloned();
5410        let scratch = self.state.dir().join("base-sync");
5411        if git::rebase_in_progress(&scratch).await {
5412            // Older resource stops predate operation reservations. Import the
5413            // standing Git operation, never retarget it to today's main.
5414            if pending.is_none()
5415                && self.state.base_sync_recovery.operations.is_empty()
5416                && !conflicted
5417                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5418                    .await
5419                    .ok()
5420                    .as_deref()
5421                    == Some(head.as_str())
5422                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5423                && git::is_clean(&winner.worktree).await?
5424            {
5425                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5426                if self.state.base_sync.is_none() {
5427                    self.state.base_sync = Some(BaseSync {
5428                        tip: saved_tip.clone(),
5429                        behind: 0,
5430                        attempts: 0,
5431                        conflict: None,
5432                        already_in: None,
5433                    });
5434                }
5435                self.state.reserve_base_operation(&head, &saved_tip);
5436                self.state.event(
5437                    "base-recovery",
5438                    "imported standing legacy rebase as a spent synchronization reservation",
5439                );
5440                self.state.save()?;
5441                pending = self.state.base_sync_recovery.operations.last().cloned();
5442            }
5443            let expected = pending.as_ref().map(|op| op.tip.as_str());
5444            if expected.is_none()
5445                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5446            {
5447                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5448                if self.state.base_sync.is_none() {
5449                    self.state.base_sync = Some(BaseSync {
5450                        tip: tip.clone(),
5451                        behind: 0,
5452                        attempts: 0,
5453                        conflict: None,
5454                        already_in: None,
5455                    });
5456                }
5457                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5458                self.state.event("land", why);
5459                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5460                    .await?;
5461                return Ok(());
5462            }
5463        }
5464        if let Some(op) = &pending {
5465            // A replay may add the pinned base's commits, never foreign work.
5466            let mut replay_proven =
5467                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5468            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5469                let wait = repository_wait(&self.state, None);
5470                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5471                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5472                    Ok(true) => {
5473                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5474                        self.state.save()?;
5475                        git::worktree_remove(&repo, &scratch).await.ok();
5476                        replay_proven = true;
5477                    }
5478                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5479                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5480                }
5481            }
5482            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5483                if self.state.foreign_checkout.is_some() {
5484                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5485                        .await
5486                        .ok()
5487                        .as_deref()
5488                        == Some(op.head.as_str())
5489                        && git::rev_parse(&repo, &format!("{head}^2"))
5490                            .await
5491                            .ok()
5492                            .as_deref()
5493                            == Some(op.tip.as_str());
5494                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5495                    .await
5496                    .is_ok_and(|m| m.is_empty())
5497                    && let Ok(added) =
5498                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5499                {
5500                    replay_proven = true;
5501                    for commit in added {
5502                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5503                            replay_proven = false;
5504                            break;
5505                        }
5506                    }
5507                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5508                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5509                }
5510            }
5511            // Never retarget an operation across a crash, even if main moved.
5512            // sync_to_head resets and cleans with -x, so even ignored untracked
5513            // files must stop recovery independently of the old tracked tree.
5514            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5515                || (head != op.head
5516                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5517                        .await?
5518                        .is_empty())
5519                || (!git::is_clean(&winner.worktree).await?
5520                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5521                        .await
5522                        .is_ok_and(|o| o.ok()))
5523                || !replay_proven
5524            {
5525                let why = format!(
5526                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5527                    op.id
5528                );
5529                let sync = self
5530                    .state
5531                    .base_sync
5532                    .as_mut()
5533                    .context("reserved synchronization has no lifetime accounting")?;
5534                sync.conflict = Some(why.clone());
5535                self.state.event("land", why);
5536                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5537                    .await?;
5538                return Ok(());
5539            }
5540            if head != op.head {
5541                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5542            }
5543            tip = op.tip.clone();
5544        }
5545        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5546        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5547        if pending.is_none()
5548            && !conflicted
5549            && fetched_base
5550            && self
5551                .state
5552                .base_sync_recovery
5553                .episodes
5554                .last()
5555                .is_some_and(|e| e.boundary.is_some())
5556            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5557            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5558            && owner.runs.last() == Some(&self.state.id)
5559            && owner.status != crate::queue::TaskStatus::Done
5560            && !owner.fresh_start
5561            && owner.review_branch.is_none()
5562            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5563            && let Some(approval) = owner.resumption.as_ref()
5564            && approval.instruction == owner.instruction
5565            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5566            && git::is_clean(&winner.worktree).await?
5567            && self.state.renew_base_episode(&task, approval, &head, &tip)
5568        {
5569            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5570            self.state.save()?;
5571        }
5572
5573        // Before any rebase, and before a recorded conflict is honoured: a
5574        // branch whose change reached the base under other commit ids has
5575        // nothing to rebase and nothing to conflict with, and a run that
5576        // already stopped on that phantom conflict recovers here on resume.
5577        // `behind == 0` with head == tip is a branch the base has since taken
5578        // in whole, whether or not a conflict was ever recorded: the ancestry
5579        // proof must still run (`classify` ignores a head still on the start
5580        // commit).
5581        if (behind > 0 || conflicted || head == tip)
5582            && self
5583                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5584                .await?
5585        {
5586            return Ok(());
5587        }
5588        if conflicted && self.state.base_recovery_pending() {
5589            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5590            let owner =
5591                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5592            let owner_valid = owner.as_ref().is_ok_and(|t| {
5593                t.runs.last() == Some(&self.state.id)
5594                    && t.status != crate::queue::TaskStatus::Done
5595                    && !t.fresh_start
5596                    && t.review_branch.is_none()
5597            });
5598            let invalidation = if !owner_valid {
5599                Some("latest task ownership changed")
5600            } else if !owner.as_ref().is_ok_and(|t| {
5601                authorization
5602                    .generation
5603                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5604            }) {
5605                Some("owner approval generation changed")
5606            } else if let Some(scope) = &authorization.scope {
5607                if scope.tip != tip {
5608                    Some("fetched base changed")
5609                } else if !self
5610                    .state
5611                    .base_sync_recovery
5612                    .episodes
5613                    .last()
5614                    .is_some_and(|e| e.id == scope.episode)
5615                {
5616                    Some("active synchronization interval changed")
5617                } else if !owner
5618                    .as_ref()
5619                    .is_ok_and(|t| t.instruction == scope.instruction)
5620                {
5621                    Some("task instruction changed")
5622                } else {
5623                    None
5624                }
5625            } else {
5626                None
5627            };
5628            if let Some(reason) = invalidation {
5629                let authorization = self
5630                    .state
5631                    .base_sync_recovery
5632                    .authorizations
5633                    .last_mut()
5634                    .unwrap();
5635                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5636                self.state.save()?;
5637                return Ok(());
5638            }
5639            if !fetched_base
5640                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5641                || !git::is_clean(&winner.worktree).await?
5642                || !owner.is_ok_and(|t| t.runs.last() == Some(&self.state.id))
5643            {
5644                return Ok(());
5645            }
5646        }
5647        let recovery = conflicted && self.state.consume_base_recovery(&head);
5648        if conflicted {
5649            // Consumption (including a refused changed HEAD) is durable before Git.
5650            self.state.save()?;
5651            if !recovery {
5652                return Ok(());
5653            }
5654        }
5655
5656        if behind == 0 {
5657            // A fixer-finished rebase moves the branch ref before the
5658            // winner's worktree is told (`sync_to_head` below). A run that
5659            // died in between resumes here with `behind == 0` and a tree still
5660            // holding the pre-rebase files, which review and the gate would
5661            // then read. That state is exactly: HEAD moved off the tip the
5662            // rebase started from, yet the tree is still identical to that
5663            // tip. A tree with edits of its own differs from it, so nothing
5664            // is thrown away.
5665            if let Some(from) = self
5666                .state
5667                .rebase_fixes
5668                .iter()
5669                .rev()
5670                .find_map(|r| r.from.clone())
5671                && from != head
5672                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5673                    .await
5674                    .is_ok_and(|o| o.ok())
5675            {
5676                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5677            }
5678            // An earlier attempt may have rebased the branch locally and died
5679            // before pushing it (only the fresh-rebase arm below pushes).
5680            // Publish it now, so the plain push at PR time is not refused as
5681            // a non-fast-forward. A run already holding a recorded conflict
5682            // never reaches here; that case is out of scope.
5683            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5684            if let Some(why) = &conflict {
5685                self.state.status = RunStatus::Blocked;
5686                self.state.event("land", why.clone());
5687            }
5688            self.state.base_sync = Some(BaseSync {
5689                tip,
5690                behind: 0,
5691                attempts,
5692                conflict,
5693                already_in: None,
5694            });
5695            if pending.is_some() {
5696                self.state.finish_base_operation(Some(&head));
5697            }
5698            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5699                .await?;
5700            self.state.save()?;
5701            return Ok(());
5702        }
5703
5704        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5705            let why = format!(
5706                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5707                 synchronization(s); synchronizing again would only race it",
5708                winner.branch
5709            );
5710            self.state.status = RunStatus::Blocked;
5711            self.state.base_sync = Some(BaseSync {
5712                tip,
5713                behind,
5714                attempts,
5715                conflict: Some(why.clone()),
5716                already_in: None,
5717            });
5718            self.state.event("land", why);
5719            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5720                .await?;
5721            self.state.save()?;
5722            return Ok(());
5723        }
5724
5725        if self.state.foreign_checkout.is_some() {
5726            // Delivery to the operator's checkout is fast-forward only. A
5727            // rebase here would discard its saved head from our ancestry.
5728            let wait = repository_wait(&self.state, None);
5729            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5730            if self.state.base_sync.is_none() {
5731                self.state.base_sync = Some(BaseSync {
5732                    tip: tip.clone(),
5733                    behind,
5734                    attempts: 0,
5735                    conflict: None,
5736                    already_in: None,
5737                });
5738            }
5739            self.state.reserve_base_operation(&head, &tip);
5740            self.state.save()?;
5741            let merged = git::git_raw(
5742                &winner.worktree,
5743                &[
5744                    "-c",
5745                    "core.hooksPath=/dev/null",
5746                    "merge",
5747                    "--no-ff",
5748                    "--no-autostash",
5749                    "--no-edit",
5750                    "-m",
5751                    &format!("Merge {tracking} before verifying"),
5752                    &tip,
5753                ],
5754            )
5755            .await?;
5756            let conflict = if merged.ok() {
5757                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5758                self.state.event(
5759                    "land",
5760                    format!("merged {tracking} into {} before verifying", winner.branch),
5761                );
5762                None
5763            } else {
5764                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5765                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5766                }
5767                let why = format!(
5768                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5769                    winner.branch,
5770                    merged.stderr.chars().take(600).collect::<String>()
5771                );
5772                self.state.status = RunStatus::Blocked;
5773                self.state.event("land", &why);
5774                Some(why)
5775            };
5776            let synced = conflict.is_none();
5777            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5778            self.state
5779                .finish_base_operation(synced.then_some(fresh.as_str()));
5780            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5781            self.state.base_sync = Some(BaseSync {
5782                tip,
5783                behind: if synced { 0 } else { behind },
5784                attempts,
5785                conflict,
5786                already_in: None,
5787            });
5788            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5789                .await?;
5790            self.state.save()?;
5791            return Ok(());
5792        }
5793
5794        self.state.event(
5795            "land",
5796            format!(
5797                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5798                winner.branch
5799            ),
5800        );
5801        self.state.save()?;
5802
5803        // The remote's copy of the branch, read now and only if the fetch
5804        // really succeeded (a stale tracking ref must never pin a lease). It is
5805        // pushed over after a rebase only when it is a commit this branch
5806        // already contains, by ancestry or by patch (an earlier rebase of ours
5807        // that never reached the remote): anything else is somebody else's work.
5808        let branch_tracking = format!("{remote}/{}", winner.branch);
5809        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5810        let remote_tip = if self.state.task_pr_branch().is_none()
5811            && matches!(&fetched_branch, Ok(o) if o.ok())
5812        {
5813            git::rev_parse(&repo, &branch_tracking).await.ok()
5814        } else {
5815            None
5816        };
5817        // A remote tip this branch does not contain is somebody else's work:
5818        // rebasing would leave a local tip that can never be pushed. Stop
5819        // before touching anything and say so.
5820        if let Some(theirs) = &remote_tip
5821            && !git::is_ancestor(&repo, theirs, &head).await
5822            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5823                .await
5824                .is_ok_and(|missing| missing.is_empty())
5825        {
5826            let why = format!(
5827                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5828                 them",
5829                short(theirs),
5830                winner.branch
5831            );
5832            self.state.status = RunStatus::Blocked;
5833            self.state.base_sync = Some(BaseSync {
5834                tip,
5835                behind,
5836                attempts,
5837                conflict: Some(why.clone()),
5838                already_in: None,
5839            });
5840            self.state.event("land", why);
5841            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5842                .await?;
5843            self.state.save()?;
5844            return Ok(());
5845        }
5846
5847        if self.state.base_sync.is_none() {
5848            self.state.base_sync = Some(BaseSync {
5849                tip: tip.clone(),
5850                behind,
5851                attempts: 0,
5852                conflict: None,
5853                already_in: None,
5854            });
5855        }
5856        self.state.reserve_base_operation(&head, &tip);
5857        self.state.save()?;
5858        let scratch = self.state.dir().join("base-sync");
5859        let rebased =
5860            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5861                .await
5862            {
5863                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5864                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5865                Err(e) => Err(e),
5866            };
5867        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5868        let stop_kind = match &rebased {
5869            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5870            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5871            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5872        };
5873        match rebased {
5874            Ok(None) => {
5875                // The branch ref moved, but a worktree that already had it
5876                // checked out (the winner's) was not told; sync its index and
5877                // files before anything reads them.
5878                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5879                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5880                self.state.finish_base_operation(Some(&fresh));
5881                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5882                let mut conflict = None;
5883                if let Some(pinned) = &remote_tip {
5884                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5885                    match pushed {
5886                        Ok(o) if o.ok() => self.state.event(
5887                            "land",
5888                            format!("pushed rebased {} to {remote}", winner.branch),
5889                        ),
5890                        Ok(o) => {
5891                            conflict = Some(format!(
5892                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5893                                winner.branch,
5894                                short(pinned),
5895                                o.stderr.chars().take(600).collect::<String>()
5896                            ));
5897                        }
5898                        Err(e) => {
5899                            conflict = Some(format!(
5900                                "rebased {} locally but could not push it: {e:#}",
5901                                winner.branch
5902                            ));
5903                        }
5904                    }
5905                }
5906                if let Some(why) = &conflict {
5907                    self.state.status = RunStatus::Blocked;
5908                    self.state.event("land", why.clone());
5909                }
5910                self.state.base_sync = Some(BaseSync {
5911                    tip: tip.clone(),
5912                    behind: 0,
5913                    attempts,
5914                    conflict,
5915                    already_in: None,
5916                });
5917                self.state
5918                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5919            }
5920            Ok(Some(conflict)) => {
5921                self.state.finish_base_operation(None);
5922                let why = format!(
5923                    "{} conflicts with {tracking} and did not rebase: {}",
5924                    winner.branch,
5925                    conflict.chars().take(600).collect::<String>()
5926                );
5927                self.state.status = RunStatus::Blocked;
5928                self.state.base_sync = Some(BaseSync {
5929                    tip,
5930                    behind,
5931                    attempts,
5932                    conflict: Some(why.clone()),
5933                    already_in: None,
5934                });
5935                self.state.event("land", why);
5936            }
5937            // A refused Git write or a busy repository: nothing is wrong with
5938            // the branch. The scratch rebase stays standing and the run
5939            // resumes into it (`base_sync` must not claim a conflict).
5940            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5941                self.state.save()?;
5942                return Ok(());
5943            }
5944            Err(e) => {
5945                self.state.finish_base_operation(None);
5946                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5947                self.state.status = RunStatus::Blocked;
5948                self.state.base_sync = Some(BaseSync {
5949                    tip,
5950                    behind,
5951                    attempts,
5952                    conflict: Some(why.clone()),
5953                    already_in: None,
5954                });
5955                self.state.event("land", why);
5956            }
5957        }
5958        if !self.state.base_sync_blocked() {
5959            self.state.base_sync_recovery.stop = None;
5960        }
5961        self.hold_base_sync_kind(stop_kind).await?;
5962        self.state.save()?;
5963        Ok(())
5964    }
5965
5966    /// Push a branch an earlier attempt rebased locally but never published,
5967    /// pinned to the remote tip read right after a successful fetch. Returns
5968    /// the reason when the run must stop; `None` when there was nothing to do
5969    /// (no remote copy, the same tip, or a remote copy this branch already
5970    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5971    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5972        // Task competitions publish only after the PR text gate, to their
5973        // public head; never resurrect a candidate branch on the remote.
5974        if self.state.task_pr_branch().is_some() {
5975            return None;
5976        }
5977        let repo = self.state.repo.clone();
5978        let remote = self.state.config.merge.remote.clone();
5979        let fetched = git::fetch(&repo, &remote, branch).await;
5980        if !matches!(&fetched, Ok(o) if o.ok()) {
5981            return None;
5982        }
5983        let branch_tracking = format!("{remote}/{branch}");
5984        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5985        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5986            return None;
5987        }
5988        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5989            .await
5990            .is_ok_and(|missing| missing.is_empty())
5991        {
5992            return Some(format!(
5993                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5994                 them",
5995                short(&theirs)
5996            ));
5997        }
5998        match git::push_pinned(&repo, &remote, branch, &theirs).await {
5999            Ok(o) if o.ok() => {
6000                self.state
6001                    .event("land", format!("pushed rebased {branch} to {remote}"));
6002                None
6003            }
6004            Ok(o) => Some(format!(
6005                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6006                 someone may have pushed): {}",
6007                short(&theirs),
6008                o.stderr.chars().take(600).collect::<String>()
6009            )),
6010            Err(e) => Some(format!(
6011                "{branch} is rebased locally but could not be pushed: {e:#}"
6012            )),
6013        }
6014    }
6015
6016    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6017    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6018    /// whether it did.
6019    ///
6020    /// Checked only when the base is ahead of the branch. A failing check is
6021    /// "not proven" - the ordinary rebase path then decides - never a reason to
6022    /// stop the run.
6023    ///
6024    /// The remote copy of the branch is held to the same standard as the local
6025    /// one: if it carries a tip this worktree does not, that tip must itself be
6026    /// proven in the base, or nothing is settled (a pull request would
6027    /// otherwise be closed over commits nobody checked). The pull request is
6028    /// closed *before* the terminal status is saved; if that fails for a
6029    /// reason other than a refusal (no network, a `gh` error) the run is left
6030    /// `Blocked` with the reason as its conflict, which a resume retries -
6031    /// the same recovery a phantom conflict gets.
6032    async fn settle_already_in(
6033        &mut self,
6034        branch: &str,
6035        tip: &str,
6036        head: &str,
6037        attempts: usize,
6038        behind: usize,
6039    ) -> Result<bool> {
6040        let repo = self.state.repo.clone();
6041        let remote = self.state.config.merge.remote.clone();
6042        let start = self.state.base_commit.clone();
6043        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6044            Ok(Some(e)) => e,
6045            Ok(None) => return Ok(false),
6046            Err(e) => {
6047                tracing::warn!("already-in-base check for {branch}: {e:#}");
6048                return Ok(false);
6049            }
6050        };
6051        let mut verified = vec![head.to_owned()];
6052        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6053        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6054        if matches!(&fetched, Ok(o) if o.ok())
6055            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6056            && theirs != head
6057        {
6058            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6059                Ok(Some(_)) => verified.push(theirs),
6060                _ => return Ok(false),
6061            }
6062        }
6063        let base_branch = self.state.base_branch.clone();
6064        let message = format!(
6065            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6066             land",
6067            evidence.names(),
6068            evidence.proof.as_str()
6069        );
6070        let closed =
6071            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6072                .await;
6073        match closed {
6074            Ok(Ok(url)) => self
6075                .state
6076                .event("land", format!("closed {url}: superseded on {base_branch}")),
6077            Ok(Err(why)) => self
6078                .state
6079                .event("land", format!("did not close a pull request: {why}")),
6080            Err(e) => {
6081                let why = format!(
6082                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6083                     not be closed ({e:#}); resume to retry"
6084                );
6085                self.state.status = RunStatus::Blocked;
6086                self.state.base_sync = Some(BaseSync {
6087                    tip: tip.to_owned(),
6088                    behind,
6089                    attempts,
6090                    conflict: Some(why.clone()),
6091                    already_in: None,
6092                });
6093                self.state.event("land", why);
6094                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6095                    .await?;
6096                self.state.save()?;
6097                return Ok(true);
6098            }
6099        }
6100        self.state.status = RunStatus::AlreadyInBase;
6101        self.state.base_sync_recovery.stop = None;
6102        self.state.base_sync = Some(BaseSync {
6103            tip: tip.to_owned(),
6104            behind,
6105            attempts,
6106            conflict: None,
6107            already_in: Some(evidence),
6108        });
6109        self.state.event("land", message);
6110        self.state.save()?;
6111        self.settle_questions();
6112        Ok(true)
6113    }
6114
6115    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6116    /// last landed the winner on, once it has run, else the commit the run
6117    /// branched from.
6118    ///
6119    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6120    /// and `vote` all happen before there is a winner to rebase, so they
6121    /// compare every candidate against the branch point on purpose, and a
6122    /// base that moves after they are already done cannot change an answer
6123    /// they already gave.
6124    fn landing_base(&self) -> String {
6125        self.state
6126            .base_sync
6127            .as_ref()
6128            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6129    }
6130
6131    // ------------------------------------------------------- operator fix
6132
6133    /// Route specific, already-recorded review findings to a fixer for a
6134    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6135    /// entry point.
6136    ///
6137    /// Distinct from `review_loop`'s own fix step in three ways: it never
6138    /// runs a reviewer wave, it never spends review-round budget, and what
6139    /// happened is recorded as an [`OperatorFixRequest`] appended to
6140    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6141    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6142    /// and vote must never be rewritten to look like a manufactured blocking
6143    /// verdict.
6144    ///
6145    /// Only meaningful once review has actually concluded: `Ready` (handed
6146    /// off with findings still open, or simply concluded clean while minor
6147    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6148    /// gate failed). Everything else is refused: a run still in progress
6149    /// should simply be resumed, and a `Merged` run's branch has already
6150    /// landed — reopening *this* run's own record cannot change that, so the
6151    /// answer there is a fresh `magi review <branch>`.
6152    ///
6153    /// A real commit here re-verifies through a fresh, ordinary review-only
6154    /// run on the same branch ([`Self::review`]) rather than reopening this
6155    /// run's own `review_loop`: once any round in this run's history went
6156    /// clean, `review_conclusion` treats that as permanent by design (the
6157    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6158    /// way to force one more genuine reviewer wave out of *this* run without
6159    /// either rewriting history or weakening that guarantee for every other
6160    /// caller. A review-only run costs nothing extra — no implementation, no
6161    /// judging, no vote — and exercises the exact same review → verify →
6162    /// gate → (human) merge path, unmodified.
6163    pub async fn fix_selected(
6164        &mut self,
6165        ids: &[String],
6166        reason: &str,
6167        allow_stale: bool,
6168    ) -> Result<()> {
6169        let reason = reason.trim();
6170        if reason.is_empty() {
6171            bail!("a fix request needs a reason — that is the operator's own record of why");
6172        }
6173        if ids.is_empty() {
6174            bail!("no finding id given");
6175        }
6176        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6177            bail!(
6178                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6179                 has already concluded — can be given a targeted fix. A run still \
6180                 in progress should simply be resumed; a `merged` run's branch has \
6181                 already landed, so its answer is a fresh `magi review <branch>`, \
6182                 not reopening this run's own record",
6183                self.state.id,
6184                self.state.status.as_str()
6185            );
6186        }
6187        let Some(winner) = self.state.winner().cloned() else {
6188            bail!("run {} has no winning candidate to fix", self.state.id);
6189        };
6190        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6191            bail!(
6192                "branch `{}` no longer exists; this run cannot be extended",
6193                winner.branch
6194            );
6195        }
6196        let home = crate::run::home();
6197        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6198            bail!(
6199                "run {} is currently being worked on by another magi process",
6200                self.state.id
6201            );
6202        }
6203        // Held for the rest of this call, including the follow-up review
6204        // below: two `magi fix` invocations against the same run must not
6205        // both reach the worktree manipulation further down, which would
6206        // otherwise race to remove and recreate the same directory — see
6207        // [`FixClaim`]'s own doc.
6208        let _claim = FixClaim::acquire(&self.state.dir())?;
6209
6210        // Resolve every id before spending anything — an unknown id refuses
6211        // the whole request rather than silently dropping it — and dedup
6212        // while keeping the operator's own order.
6213        let mut seen = BTreeSet::new();
6214        let mut findings = Vec::new();
6215        let mut missing = Vec::new();
6216        for id in ids {
6217            if !seen.insert(id.clone()) {
6218                continue;
6219            }
6220            match self.state.finding(id) {
6221                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6222                    id: f.id.clone(),
6223                    severity: f.severity,
6224                    reviewer_vote: rec.vote,
6225                    round: round.round,
6226                    round_head: round.head.clone(),
6227                    reviewer: rec.reviewer,
6228                    agent: rec.agent.clone(),
6229                    file: f.file.clone(),
6230                    line: f.line,
6231                    title: f.title.clone(),
6232                    detail: f.detail.clone(),
6233                    outcome: OperatorFixOutcome::Pending,
6234                }),
6235                None => missing.push(id.clone()),
6236            }
6237        }
6238        if !missing.is_empty() {
6239            bail!(
6240                "unknown finding id(s): {}; nothing was changed",
6241                missing.join(", ")
6242            );
6243        }
6244
6245        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6246        let stale_details: Vec<(String, String)> = findings
6247            .iter()
6248            .filter(|f| f.round_head != head_at_request)
6249            .map(|f| (f.id.clone(), f.round_head.clone()))
6250            .collect();
6251        let stale = !stale_details.is_empty();
6252        if stale && !allow_stale {
6253            bail!(
6254                "the branch has moved since some finding(s) were raised — {} — now \
6255                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6256                stale_details
6257                    .iter()
6258                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6259                    .collect::<Vec<_>>()
6260                    .join(", "),
6261                short(&head_at_request)
6262            );
6263        }
6264
6265        let request = OperatorFixRequest {
6266            requested_at: Timestamp::now(),
6267            reason: reason.to_owned(),
6268            findings,
6269            head_at_request: head_at_request.clone(),
6270            allow_stale,
6271            stale,
6272            fix: None,
6273            result_head: None,
6274            follow_up_review_run: None,
6275        };
6276        self.state.event(
6277            "fix",
6278            format!(
6279                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6280                request.findings.len(),
6281                request
6282                    .findings
6283                    .iter()
6284                    .map(|f| f.id.as_str())
6285                    .collect::<Vec<_>>()
6286                    .join(", "),
6287            ),
6288        );
6289        // Recorded now, before any worktree work or the fixer call itself —
6290        // and re-saved at each checkpoint below: a crash at any point after
6291        // this (mid fixer call, mid follow-up review) must not lose the fact
6292        // that this was requested, for which findings, and why. Everything
6293        // past this point reads and writes through `request_index` rather
6294        // than a local variable, since `request` itself is moved here.
6295        self.state.operator_fixes.push(request);
6296        self.state.save()?;
6297        let request_index = self.state.operator_fixes.len() - 1;
6298
6299        // A fresh, dedicated worktree for this one call, never the winner's
6300        // own worktree in place: that one may already be gone (folded away),
6301        // and reusing it in place would leave the branch checked out there
6302        // when the follow-up review below tries to check it out again. Freed
6303        // immediately after, either way — but only once confirmed clean:
6304        // `worktree_remove` is a `git worktree remove --force`, which would
6305        // otherwise discard uncommitted work left there by the operator or
6306        // another process before this had a chance to even look at it.
6307        let repo = self.state.repo.clone();
6308        let coordination = self.coordinate(&repo).await?;
6309        if winner.worktree.exists() {
6310            // Lockfiles a rescue commit withheld stay untracked on purpose and
6311            // are already recorded; they are not the operator's work to protect.
6312            let dirty = git::git(
6313                &winner.worktree,
6314                &["status", "--porcelain", "--untracked-files=all"],
6315            )
6316            .await?;
6317            let only_withheld = dirty.lines().all(|l| {
6318                l.strip_prefix("?? ")
6319                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6320            });
6321            if !only_withheld {
6322                bail!(
6323                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6324                     discard them first",
6325                    winner.worktree.display()
6326                );
6327            }
6328            git::worktree_remove(&self.state.repo, &winner.worktree)
6329                .await
6330                .ok();
6331        }
6332        let fix_worktree = self.state.worktree_root().join("operator-fix");
6333        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6334        git::git(
6335            &self.state.repo,
6336            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6337        )
6338        .await
6339        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6340        if !git::is_clean(&fix_worktree).await? {
6341            git::worktree_remove(&self.state.repo, &fix_worktree)
6342                .await
6343                .ok();
6344            bail!(
6345                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6346                winner.branch
6347            );
6348        }
6349        let repo_for_setup = self.state.repo.clone();
6350        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6351            Ok(()) => {
6352                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6353            }
6354            Err(e) => Err(e),
6355        };
6356        if let Err(e) = setup {
6357            git::worktree_remove(&self.state.repo, &fix_worktree)
6358                .await
6359                .ok();
6360            return Err(e);
6361        }
6362
6363        // This explicit operator request retries availability independently
6364        // of execute(); the previous call has already saved and cleaned up.
6365        self.state.resource_block = None;
6366
6367        let run_id = self.state.id.clone();
6368        let prompts = self.state.config.prompts.clone();
6369        let language = self.state.config.graph.language.clone();
6370        let sessions = self.state.config.graph.sessions;
6371        let artifacts = agent::artifacts_dir(&self.state.dir());
6372        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6373            .findings
6374            .iter()
6375            .map(|f| Finding {
6376                id: f.id.clone(),
6377                severity: f.severity,
6378                file: f.file.clone(),
6379                line: f.line,
6380                title: f.title.clone(),
6381                detail: f.detail.clone(),
6382            })
6383            .collect();
6384        let fix_prompt = prompt::operator_fix_in(
6385            &self.state.instruction,
6386            &finding_list,
6387            reason,
6388            &stale_details,
6389            &head_at_request,
6390            &language,
6391            &self.state.config.graph.github_language,
6392        );
6393        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6394        drop(coordination);
6395        let (job, seat, out) = self
6396            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6397                prompt: fix_prompt.clone(),
6398                spec,
6399                seat,
6400                cwd: fix_worktree.clone(),
6401                timeout,
6402                allow_write: true,
6403                sessions,
6404                artifacts: artifacts.clone(),
6405                stem: "operator-fix".to_owned(),
6406                handover: None,
6407            })
6408            .await;
6409        let agent_id = seat.agent.clone();
6410
6411        let mut fix = FixRecord {
6412            agent: agent_id,
6413            addressed: Vec::new(),
6414            rejected: Vec::new(),
6415            notes: String::new(),
6416            committed: false,
6417            failed: None,
6418            duration_ms: 0,
6419            continuation: None,
6420        };
6421        let mut final_seat = seat.clone();
6422        match out {
6423            AgentOutcome::Ok(o) => {
6424                fix.duration_ms = o.duration_ms;
6425                let parsed = verdict::extract_json::<FixReport>(&o.text);
6426                let incomplete_reason = match &parsed {
6427                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6428                        "the reply parsed, but it reported a command whose own CLI \
6429                         never confirmed an exit status"
6430                            .to_owned(),
6431                    ),
6432                    Ok(_) => None,
6433                    Err(e) => Some(e.to_string()),
6434                };
6435                match incomplete_reason {
6436                    None => {
6437                        let report = parsed.expect("checked Ok above");
6438                        fix.addressed = report.addressed;
6439                        fix.rejected = report.rejected;
6440                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6441                    }
6442                    Some(reason) => {
6443                        let (resumed_seat, resolved, failure, cont) = self
6444                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6445                            .await;
6446                        fix.duration_ms += cont.cumulative_wait_ms;
6447                        fix.continuation = Some(cont);
6448                        final_seat = resumed_seat;
6449                        match resolved {
6450                            Some(report) => {
6451                                fix.addressed = report.addressed;
6452                                fix.rejected = report.rejected;
6453                                fix.notes =
6454                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6455                            }
6456                            None => fix.failed = failure,
6457                        }
6458                    }
6459                }
6460            }
6461            AgentOutcome::Dropped(o) => {
6462                fix.duration_ms = o.duration_ms;
6463                let why = o
6464                    .dropped
6465                    .as_ref()
6466                    .map(|d| d.why.as_str())
6467                    .unwrap_or("the CLI ended the stream without delivering its answer");
6468                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6469            }
6470            AgentOutcome::Quota(o) => {
6471                self.state.quota.push(QuotaLoss::from_report(
6472                    final_seat.key.clone(),
6473                    "fix",
6474                    o.quota.as_ref(),
6475                ));
6476                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6477            }
6478            AgentOutcome::Failed(e) => fix.failed = Some(e),
6479            AgentOutcome::ResourceBlocked(e) => {
6480                fix.failed = Some(format!("the fixer was not started: {e}"));
6481            }
6482        }
6483        // Even a deferred call must rescue edits, release the worktree and
6484        // finalize its record below so an explicit `magi fix` can retry.
6485        if fix.continuation.is_none() {
6486            fix.continuation = Some(ContinuationRecord::not_needed());
6487        }
6488        self.state.seats.insert(final_seat.key.clone(), final_seat);
6489
6490        let rescue_message = format!(
6491            "magi: operator-selected fix ({}) (uncommitted work)",
6492            self.state.operator_fixes[request_index]
6493                .findings
6494                .iter()
6495                .map(|f| f.id.as_str())
6496                .collect::<Vec<_>>()
6497                .join(", ")
6498        );
6499        let repo = self.state.repo.clone();
6500        let coordination = self.coordinate(&repo).await?;
6501        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6502            self.state.note_withheld("fix", &r.withheld);
6503        }
6504        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6505        fix.committed = after != head_at_request;
6506        git::worktree_remove(&self.state.repo, &fix_worktree)
6507            .await
6508            .ok();
6509        drop(coordination);
6510
6511        self.state.event(
6512            "fix",
6513            match &fix.failed {
6514                Some(reason) => format!(
6515                    "operator fix: adoption report was lost ({reason}); {}",
6516                    if fix.committed {
6517                        "committed"
6518                    } else {
6519                        "NO new commit"
6520                    }
6521                ),
6522                None => format!(
6523                    "operator fix: {} addressed, {} rejected, {}",
6524                    fix.addressed.len(),
6525                    fix.rejected.len(),
6526                    if fix.committed {
6527                        "committed"
6528                    } else {
6529                        "NO new commit"
6530                    }
6531                ),
6532            },
6533        );
6534
6535        // Every selected finding gets an outcome — never left `Pending` once
6536        // the fixer's own turn is over. A report that never came back at all
6537        // marks every one of them `Unreported`, not silently "not addressed":
6538        // quota, a dropped stream, or an exhausted continuation are gaps in
6539        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6540        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6541        for f in &mut self.state.operator_fixes[request_index].findings {
6542            f.outcome = if fix.failed.is_some() {
6543                OperatorFixOutcome::Unreported
6544            } else if fix.addressed.contains(&f.id) {
6545                OperatorFixOutcome::Addressed
6546            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6547                OperatorFixOutcome::Rejected { why: r.why.clone() }
6548            } else {
6549                OperatorFixOutcome::Unreported
6550            };
6551        }
6552
6553        let committed = fix.committed;
6554        if committed {
6555            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6556        }
6557        self.state.operator_fixes[request_index].fix = Some(fix);
6558        // Saved again now that the fixer's own outcome is final, on top of
6559        // the save right after the request was first pushed above.
6560        self.state.save()?;
6561        if self.state.resource_block.is_some() {
6562            return self.stop_for_resource();
6563        }
6564
6565        if committed {
6566            self.state.event(
6567                "fix",
6568                format!(
6569                    "operator fix committed {}; opening a follow-up review-only run",
6570                    short(&after)
6571                ),
6572            );
6573            // The operator asked for the fix, and the follow-up serves whatever
6574            // task the run it follows served.
6575            let origin =
6576                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6577            match Self::review(
6578                &self.state.repo,
6579                &winner.branch,
6580                self.state.config.clone(),
6581                origin,
6582            )
6583            .await
6584            {
6585                Ok(mut follow_up) => {
6586                    follow_up.state.event(
6587                        "start",
6588                        format!(
6589                            "requested by an operator fix on run {} for finding(s) {}",
6590                            self.state.id,
6591                            self.state.operator_fixes[request_index]
6592                                .findings
6593                                .iter()
6594                                .map(|f| f.id.as_str())
6595                                .collect::<Vec<_>>()
6596                                .join(", "),
6597                        ),
6598                    );
6599                    follow_up.state.save()?;
6600                    let follow_up_id = follow_up.state.id.clone();
6601                    // The follow-up is a run like any other: it belongs to the
6602                    // task of the run it follows, or to one filed for it.
6603                    let adopted = match crate::direct::adopt(
6604                        &follow_up.state,
6605                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6606                    ) {
6607                        Ok(a) => a,
6608                        Err(e) => {
6609                            // An ownerless run must not spend agent calls; it
6610                            // stays saved, and `magi run --resume` adopts it.
6611                            self.state.event(
6612                                "fix",
6613                                format!(
6614                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6615                                ),
6616                            );
6617                            self.state.save()?;
6618                            return Ok(());
6619                        }
6620                    };
6621                    let executed = follow_up.execute().await;
6622                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6623                    if let Some(a) = adopted {
6624                        a.finish(&follow_up.state, executed);
6625                    }
6626                    if let Some(e) = failure {
6627                        self.state.event(
6628                            "fix",
6629                            format!(
6630                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6631                            ),
6632                        );
6633                    }
6634                    self.state.operator_fixes[request_index].follow_up_review_run =
6635                        Some(follow_up_id);
6636                }
6637                Err(e) => {
6638                    self.state.event(
6639                        "fix",
6640                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6641                    );
6642                }
6643            }
6644            self.state.save()?;
6645        }
6646
6647        Ok(())
6648    }
6649
6650    // --------------------------------------------------------------- review
6651
6652    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6653    /// names, in order, each at most once, moving on only when the call
6654    /// advances (an error, a quota hit or nothing usable - the same decision
6655    /// point `agent::chain_advances` is for the other chained roles).
6656    ///
6657    /// Each agent gets its own seat from `Runner::seat`: the same agent
6658    /// continues its conversation (the winner's own implementer seat when it
6659    /// is the winner's author, now that the competition is over), another
6660    /// takes a fresh one so the full prompt is sent again. A handover is
6661    /// recorded under `node`, which is what makes the fallback stick for the
6662    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6663    /// is returned, so an exhausted chain reads exactly like a single failed
6664    /// fixer: one quota loss, the same wording, the same refund. An earlier
6665    /// attempt's edits are left in the tree and judged, with the final
6666    /// attempt's, by what git says afterwards.
6667    ///
6668    /// Returns the job that produced the outcome, for `continue_fix_report`.
6669    async fn ask_fixer(
6670        &mut self,
6671        winner: &Candidate,
6672        node: &'static str,
6673        round: Option<usize>,
6674        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6675    ) -> (SeatJob, SeatState, AgentOutcome) {
6676        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6677        self.ask_fixer_among(attempts, node, round, None, build)
6678            .await
6679    }
6680
6681    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6682    /// worktree and a commit: before the next agent is asked after one that
6683    /// did not satisfy the call, the worktree is put back on that commit, so
6684    /// a failed agent's partial edits are not carried into the next one.
6685    async fn ask_fixer_among(
6686        &mut self,
6687        attempts: Vec<(AgentSpec, String)>,
6688        node: &'static str,
6689        round: Option<usize>,
6690        restore: Option<(&Path, &str)>,
6691        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6692    ) -> (SeatJob, SeatState, AgentOutcome) {
6693        let run_id = self.state.id.clone();
6694        let prompts = self.state.config.prompts.clone();
6695        let cache = self.state.config.cache_dir();
6696        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6697        let mut last = None;
6698        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6699            let seat = self.seat(&key, &spec.id);
6700            let mut job = build(spec.clone(), seat);
6701            if i > 0 {
6702                job.stem = format!("{}-{}", job.stem, spec.id);
6703            }
6704            let ctx = WaveCtx {
6705                carry_seats: false,
6706                pauses: Some((&self.pause, &self.interrupt)),
6707                run: &run_id,
6708                node,
6709                prompts: &prompts,
6710                cache: cache.as_deref(),
6711                round,
6712            };
6713            let (seat, out) =
6714                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6715            // A resource block never advances the chain: the chosen agent
6716            // was not asked, so it must be the one asked again on resume.
6717            let advances = match &out {
6718                AgentOutcome::Ok(o) => agent::output_advances(o),
6719                AgentOutcome::ResourceBlocked(_) => false,
6720                _ => true,
6721            };
6722            if let Some(next) = ids.get(i + 1)
6723                && advances
6724            {
6725                self.state.seats.insert(seat.key.clone(), seat.clone());
6726                let class =
6727                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6728                let reason = match &out {
6729                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6730                    other => fail_reason(other),
6731                };
6732                // A failed agent's edits must not reach the next one, and a
6733                // restore that did not work ends the chain: the call fails
6734                // and the caller's own (confirmed) restore decides.
6735                if let Some((wt, rev)) = restore
6736                    && let Err(e) = self.graft_restore(wt, rev).await
6737                {
6738                    let why =
6739                        format!("could not restore the worktree before the next fixer: {e:#}");
6740                    tracing::warn!("{why}");
6741                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6742                        AgentOutcome::ResourceBlocked(why)
6743                    } else {
6744                        AgentOutcome::Failed(why)
6745                    };
6746                    last = Some((job, seat, out));
6747                    break;
6748                }
6749                record_handover(
6750                    &mut self.state,
6751                    node,
6752                    &seat.key,
6753                    &spec.id,
6754                    next,
6755                    &class,
6756                    &reason,
6757                );
6758                continue;
6759            }
6760            last = Some((job, seat, out));
6761            break;
6762        }
6763        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6764        if let AgentOutcome::Quota(output) = &out
6765            && node != "graft"
6766        {
6767            // Graft is best effort: its caller records the loss and restores
6768            // the pre-graft tree, then proceeds to review and gate.
6769            // Every fallback was tried. Machine outages cannot be repaired by
6770            // spending a fix round; use each caller's resumable resource path.
6771            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6772            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6773            self.state.quota.push(loss);
6774            self.state.seats.insert(seat.key.clone(), seat.clone());
6775            self.state.status = RunStatus::Blocked;
6776            return (
6777                job,
6778                seat,
6779                AgentOutcome::ResourceBlocked(
6780                    "fixer unavailable; restore agent availability and resume".into(),
6781                ),
6782            );
6783        }
6784        (job, seat, out)
6785    }
6786
6787    /// The one repair a verification-only failure earns (see
6788    /// [`crate::recovery`]), before the first review round of a run that took
6789    /// a branch over.
6790    ///
6791    /// Saved as started *before* the fixer is called, so a resume never
6792    /// repairs twice; a start found on re-entry is rolled back to the commit
6793    /// it began from. A failed or unusable fixer leaves the tree as it was and
6794    /// the review goes on over it, so the first round's own verification
6795    /// reports the failure again. The failure is never carried into a round as
6796    /// a result: verification is re-run on whatever commit this leaves.
6797    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6798        use crate::recovery::{Mode, Repair};
6799        if !self.state.reviews.is_empty() {
6800            return Ok(());
6801        }
6802        let Some(mut rec) = self.state.recovery.clone() else {
6803            return Ok(());
6804        };
6805        if rec.mode != Mode::VerifyRepair {
6806            return Ok(());
6807        }
6808        let wt = winner.worktree.clone();
6809        let mut head = git::rev_parse(&wt, "HEAD").await?;
6810        match rec.repair.clone() {
6811            Repair::Done { .. } => return Ok(()),
6812            Repair::WaitingForRestore { before } => {
6813                self.graft_restore(&wt, &before).await?;
6814                head = before;
6815                rec.repair = Repair::NotStarted;
6816                self.state.recovery = Some(rec.clone());
6817                self.state.save()?;
6818            }
6819            Repair::Started { before } => {
6820                // Interrupted: the unfinished edits are not trusted.
6821                // An unconfirmed rollback blocks the run with `Started` kept,
6822                // so a resume retries it; the tree is never reviewed as is.
6823                if let Err(e) = self.graft_restore(&wt, &before).await {
6824                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6825                        return Err(e);
6826                    }
6827                    return self
6828                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6829                        .await;
6830                }
6831                rec.repair = Repair::Done {
6832                    before: before.clone(),
6833                    after: before,
6834                    committed: false,
6835                    failed: Some("interrupted; rolled back".to_owned()),
6836                };
6837                self.state
6838                    .event("recovery", "the repair was interrupted and rolled back");
6839                self.state.recovery = Some(rec);
6840                self.state.save()?;
6841                return Ok(());
6842            }
6843            Repair::NotStarted => {}
6844        }
6845        if crate::recovery::recheck(&mut rec, &head) {
6846            self.state.event("recovery", rec.summary());
6847            self.state.recovery = Some(rec);
6848            self.state.save()?;
6849            return Ok(());
6850        }
6851        rec.repair = Repair::Started {
6852            before: head.clone(),
6853        };
6854        self.state.recovery = Some(rec.clone());
6855        self.state.save()?;
6856
6857        let language = self.state.config.graph.language.clone();
6858        let sessions = self.state.config.graph.sessions;
6859        let artifacts = agent::artifacts_dir(&self.state.dir());
6860        let fix_prompt = prompt::recovery_fix(
6861            &self.state.instruction,
6862            &rec,
6863            self.state.config.graph.review_rounds,
6864            &language,
6865        );
6866        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6867        self.state.event(
6868            "recovery",
6869            format!(
6870                "repair: the fixer starts from the failed verification on {}",
6871                short(&head)
6872            ),
6873        );
6874        let (_job, seat, out) = self
6875            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6876                prompt: fix_prompt.clone(),
6877                spec,
6878                seat,
6879                cwd: wt.clone(),
6880                timeout,
6881                allow_write: true,
6882                sessions,
6883                artifacts: artifacts.clone(),
6884                stem: "fix-recovery".to_owned(),
6885                handover: None,
6886            })
6887            .await;
6888        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6889            rec.repair = Repair::WaitingForRestore { before: head };
6890            self.state.recovery = Some(rec);
6891            return self.stop_for_resource();
6892        }
6893        let failed = match &out {
6894            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6895            AgentOutcome::Quota(o) => {
6896                self.state.quota.push(QuotaLoss::from_report(
6897                    seat.key.clone(),
6898                    "fix",
6899                    o.quota.as_ref(),
6900                ));
6901                Some("rate limited (quota); the fixer could not run".to_owned())
6902            }
6903            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6904            AgentOutcome::Failed(e) => Some(e.clone()),
6905        };
6906        self.state.seats.insert(seat.key.clone(), seat);
6907        let coordination = self.coordinate(&wt).await?;
6908        let failed = match failed {
6909            // The agent's partial edits are not kept.
6910            Some(why) => {
6911                if let Err(e) = git::restore_to(&wt, &head).await {
6912                    return self
6913                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6914                        .await;
6915                }
6916                Some(why)
6917            }
6918            None => {
6919                if let Ok(r) = git::rescue_commit(
6920                    &wt,
6921                    "magi: recovery repair of the failed verification (uncommitted work)",
6922                )
6923                .await
6924                {
6925                    self.state.note_withheld("fix", &r.withheld);
6926                }
6927                None
6928            }
6929        };
6930        let after = git::rev_parse(&wt, "HEAD").await?;
6931        drop(coordination);
6932        self.state.event(
6933            "recovery",
6934            match &failed {
6935                Some(why) => format!("repair: nothing kept ({why})"),
6936                None if after != head => {
6937                    format!("repair: committed {}", short(&after))
6938                }
6939                None => "repair: the fixer made no commit".to_owned(),
6940            },
6941        );
6942        rec.repair = Repair::Done {
6943            before: head.clone(),
6944            committed: after != head,
6945            after,
6946            failed,
6947        };
6948        self.state.recovery = Some(rec);
6949        self.state.save()?;
6950        Ok(())
6951    }
6952
6953    /// A repair whose rollback could not be confirmed: the run is blocked and
6954    /// resumable, and the unconfirmed state is never reviewed.
6955    async fn stop_recovery(
6956        &mut self,
6957        mut rec: crate::recovery::RecoveryRecord,
6958        head: &str,
6959        why: String,
6960    ) -> Result<()> {
6961        // `Started` stays recorded: a resume retries the restore.
6962        rec.repair = crate::recovery::Repair::Started {
6963            before: head.to_owned(),
6964        };
6965        self.state.recovery = Some(rec);
6966        self.state
6967            .event("recovery", format!("repair could not be undone: {why}"));
6968        self.state.status = RunStatus::Blocked;
6969        self.state.save()?;
6970        bail!("recovery repair could not be rolled back: {why}")
6971    }
6972
6973    async fn review_loop(&mut self) -> Result<()> {
6974        if self.hold_base_sync().await? {
6975            return Ok(());
6976        }
6977        loop {
6978            if self
6979                .state
6980                .base_sync
6981                .as_ref()
6982                .is_some_and(|s| s.conflict.is_some())
6983            {
6984                return Ok(());
6985            }
6986            if self.catchup_repairable() {
6987                let record = self.state.reviews.last().expect("repairable round").clone();
6988                let winner = self
6989                    .state
6990                    .winner()
6991                    .cloned()
6992                    .context("catch-up repair has no winner")?;
6993                if !self.fix_review_round(&winner, record).await? {
6994                    return Ok(());
6995                }
6996            }
6997            self.review_loop_pass().await?;
6998            if !self.catchup_repairable() {
6999                return Ok(());
7000            }
7001        }
7002    }
7003
7004    /// Derive an owed catch-up repair from durable round evidence, including
7005    /// reentry after a crash before the fixer was dispatched.
7006    fn catchup_repairable(&self) -> bool {
7007        self.state.reviews.len() < self.state.config.graph.review_rounds
7008            && self.state.reviews.last().is_some_and(|r| {
7009                !r.clean
7010                    && r.blocking == 0
7011                    && r.fix.is_none()
7012                    && r.e2e_status() == E2eStatus::Failed
7013            })
7014    }
7015
7016    async fn review_loop_pass(&mut self) -> Result<()> {
7017        // A base that would not rebase is a person's decision, not a review
7018        // round: nothing here would change the answer, and reviewers and a
7019        // fixer would be spending real budget on a tree that cannot land
7020        // regardless of what they find.
7021        if self
7022            .state
7023            .base_sync
7024            .as_ref()
7025            .is_some_and(|s| s.conflict.is_some())
7026        {
7027            return Ok(());
7028        }
7029        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7030        // agent files with `magi task add` name the run that paid for it. The
7031        // prompt overlay is cloned alongside it because the waves borrow it
7032        // while `self` is mutably borrowed by the node's own bookkeeping.
7033        let run_id = self.state.id.clone();
7034        let prompts = self.state.config.prompts.clone();
7035        let Some(winner) = self.state.winner().cloned() else {
7036            return Ok(());
7037        };
7038        let granted_review = self
7039            .state
7040            .repair_grants
7041            .last()
7042            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7043            .cloned();
7044        let fresh_review = granted_review
7045            .as_ref()
7046            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7047        let max_rounds = granted_review
7048            .as_ref()
7049            .map_or(self.state.config.graph.review_rounds, |g| {
7050                g.review_start.saturating_add(1)
7051            });
7052        // A clean round, an exhausted round budget, or a stalled tree (see
7053        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7054        // they are recorded — recomputed here, not read off `status`, so a
7055        // reentry into a run that already stopped restates the identical
7056        // verdict instead of silently handing back whatever an earlier node
7057        // in this same walk clobbered `status` to (a solo-candidate
7058        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7059        // below runs an empty range once the budget is spent, and would
7060        // otherwise fall through without touching `status` at all.
7061        if let Some(status) =
7062            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7063        {
7064            // A reentry after a crash between the last round's save and
7065            // `stop_reviewing` reaches the hand-off here, not there.
7066            if status == RunStatus::Gating {
7067                self.record_contested_handoff();
7068            }
7069            self.state.status = status;
7070            self.state.save()?;
7071            return Ok(());
7072        }
7073        self.state.status = RunStatus::Reviewing;
7074        // A last recorded round whose own verification never resolved
7075        // (`ResourceBlocked` — the shared build cache, not the patch) is
7076        // never a concluded round, whatever the round budget says: starting
7077        // a fresh round on top of it would spend a whole new reviewer wave
7078        // re-reading an unchanged patch instead of just retrying the one
7079        // check that actually needs it, and once the budget is spent the
7080        // loop below has nothing left to do at all (its range is empty).
7081        // Retry that check directly instead, exactly the same retry
7082        // `stop_reviewing` already does for its own catch-up case.
7083        if !fresh_review
7084            && self.state.reviews.last().is_some_and(|r| {
7085                r.e2e_status() == E2eStatus::ResourceBlocked
7086                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7087            })
7088        {
7089            let shell = self.state.config.shell();
7090            return self
7091                .stop_unresolved_review(
7092                    "the last round's own verification never resolved",
7093                    &shell,
7094                    &winner.worktree,
7095                )
7096                .await;
7097        }
7098
7099        if granted_review.is_none() {
7100            self.recovery_repair(&winner).await?;
7101        }
7102        if self.resource_halt()? {
7103            return Ok(());
7104        }
7105
7106        let repo = self.state.repo.clone();
7107        let root = self.state.worktree_root();
7108        let language = self.state.config.graph.language.clone();
7109        let sessions = self.state.config.graph.sessions;
7110        let artifacts = self.merge_repair_artifacts();
7111        let base = self.landing_base();
7112        let base_short = short(&base);
7113        let reviewers = self.roles.reviewers.clone();
7114        let shell = self.state.config.shell();
7115
7116        for round in (self.state.reviews.len() + 1)..=max_rounds {
7117            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7118            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7119            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7120            // The prior round's own record, already persisted — never a
7121            // hand-carried variable of just its failing output: that is
7122            // exactly what let a round's e2e result drift out of sync with
7123            // which commit it was actually about (see `SCHEMA`'s doc for
7124            // schema 8). Judged against `head`, the commit reviewers are
7125            // about to look at now, so the summary always reads as "an
7126            // earlier head" here — this round's own patch has not been
7127            // checked yet.
7128            let prev_verification = self
7129                .state
7130                .reviews
7131                .last()
7132                .and_then(|r| r.verification_summary(&head));
7133            // Only the first round of a run that took a branch over: later
7134            // rounds have this run's own history.
7135            let prior_section = match (&self.state.recovery, round) {
7136                (Some(rec), 1) if self.state.reviews.is_empty() => {
7137                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7138                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7139                            Some(crate::recovery::head_of(
7140                                &git::diff(&winner.worktree, reviewed, "HEAD")
7141                                    .await
7142                                    .unwrap_or_default(),
7143                                crate::recovery::SINCE_DIFF_MAX,
7144                            ))
7145                        }
7146                        _ => None,
7147                    };
7148                    Some(prompt::recovery_review(rec, since.as_deref()))
7149                }
7150                _ => None,
7151            };
7152
7153            // Each reviewer gets its own detached checkout of exactly this
7154            // commit: nobody can perturb the winner's tree, and the fixer can
7155            // keep working without racing a reviewer.
7156            let mut jobs = Vec::new();
7157            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7158                self.coordinate(&repo).await?
7159            } else {
7160                None
7161            };
7162            for (r, spec) in reviewers.iter().cloned().enumerate() {
7163                let wt = root.join(format!("review-{}", r + 1));
7164                // `clean -fdx` in a reset wipes setup's products, so every
7165                // round sets the seat up again.
7166                if wt.exists() {
7167                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7168                } else {
7169                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7170                        None
7171                    } else {
7172                        self.coordinate(&repo).await?
7173                    };
7174                    git::worktree_add_detached(&repo, &wt, &head).await?;
7175                }
7176                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7177                    None
7178                } else {
7179                    self.coordinate(&repo).await?
7180                };
7181                self.setup_seat_worktree(&repo, &wt, false).await?;
7182                let seat_key = format!("review-{}", r + 1);
7183                // The seat starts the round on whoever answered it last, not
7184                // on the agent the spec names, so a failure is not re-paid.
7185                let spec = pick_start_spec(
7186                    &self.roles.reviewer_roster,
7187                    spec,
7188                    self.state.seat_history.get(&seat_key),
7189                );
7190                let seat = self.seat(&seat_key, &spec.id);
7191                jobs.push(SeatJob {
7192                    prompt: prompt::review(&prompt::ReviewCtx {
7193                        instruction: &self.state.instruction,
7194                        branch: &winner.branch,
7195                        base_short: &base_short,
7196                        stat: &stat,
7197                        patch: &patch,
7198                        verification: prev_verification.as_ref(),
7199                        prior: prior_section.as_deref(),
7200                        reviewers: reviewers.len(),
7201                        round,
7202                        rounds: max_rounds,
7203                        // A review-only run has no rankings, so nothing
7204                        // competed for this patch and the reviewer is told so.
7205                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7206                        lens: Lens::for_seat(r),
7207                        language: &language,
7208                        github_language: &self.state.config.graph.github_language,
7209                    }),
7210                    spec,
7211                    seat,
7212                    cwd: wt,
7213                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7214                    allow_write: false,
7215                    sessions,
7216                    artifacts: artifacts.clone(),
7217                    stem: format!("review-{round}-{}", r + 1),
7218                    handover: None,
7219                });
7220            }
7221
7222            drop(coordination);
7223            self.state.event(
7224                "review",
7225                format!(
7226                    "round {round}: {} reviewers on {}",
7227                    jobs.len(),
7228                    short(&head)
7229                ),
7230            );
7231            let mut quota_losses = Vec::new();
7232            let review_retries = self.state.config.graph.retries;
7233            let review_cache = self.state.config.cache_dir();
7234            let ctx = WaveCtx {
7235                carry_seats: true,
7236                pauses: Some((&self.pause, &self.interrupt)),
7237                run: &run_id,
7238                node: "review",
7239                prompts: &prompts,
7240                cache: review_cache.as_deref(),
7241                round: Some(round),
7242            };
7243            let results = ask_json_wave::<Review>(
7244                jobs,
7245                Arc::clone(&self.sem),
7246                review_retries,
7247                &self.roles.reviewer_roster,
7248                &ctx,
7249                &mut quota_losses,
7250                &mut self.state,
7251                &|_: &Review| Ok(()),
7252            )
7253            .await;
7254            if self.state.resource_block.is_some() {
7255                return self.stop_for_resource();
7256            }
7257            // Counted before the move below: how many of *this* round's
7258            // reviewer seats were lost to their own rate limit, as opposed to
7259            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7260            let round_quota_missing = quota_losses.len();
7261            self.state.quota.extend(quota_losses);
7262
7263            let mut records = Vec::new();
7264            let mut all_findings = Vec::new();
7265            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7266                let agent_id = seat.agent.clone();
7267                self.state.seats.insert(seat.key.clone(), seat);
7268                let mut record = ReviewRecord {
7269                    reviewer: r + 1,
7270                    agent: agent_id,
7271                    summary: String::new(),
7272                    findings: Vec::new(),
7273                    vote: None,
7274                    failed: None,
7275                    duration_ms: 0,
7276                    // Set for both outcomes: `failed: Some(_)` with
7277                    // `attempts > 0` is a seat every retry still lost, not a
7278                    // recovered one — only `failed: None` with `attempts > 0`
7279                    // reads as "answered after a nudge" (see this field's own
7280                    // doc).
7281                    attempts,
7282                };
7283                match res {
7284                    Ok((review, out)) => {
7285                        // Sanitized here, at the point every other piece of
7286                        // agent prose in this file is (candidate summaries,
7287                        // deliberation turns, vote reasons): a reviewer's own
7288                        // words are the one thing about it that could name
7289                        // it, and reconsideration below broadcasts this same
7290                        // summary and these same findings to every other
7291                        // seat on the panel.
7292                        record.summary =
7293                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7294                        record.vote = Some(review.vote);
7295                        record.duration_ms = out.duration_ms;
7296                        for (n, mut f) in review.findings.into_iter().enumerate() {
7297                            // ids are magi's, never the agent's: the fixer's
7298                            // adoption report is keyed by them.
7299                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7300                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7301                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7302                            // `file` is agent-supplied prose too, never
7303                            // checked against the real tree — the same
7304                            // exposure `title`/`detail` above have, just in
7305                            // a field easy to forget because it looks like a
7306                            // path rather than free text.
7307                            f.file = f
7308                                .file
7309                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7310                            all_findings.push(f.clone());
7311                            record.findings.push(f);
7312                        }
7313                        self.state.event(
7314                            "review",
7315                            format!(
7316                                "round {round}: reviewer {} voted {} with {} finding(s)",
7317                                r + 1,
7318                                review.vote.label(),
7319                                record.findings.len()
7320                            ),
7321                        );
7322                    }
7323                    Err(e) => {
7324                        record.failed = Some(e.to_string());
7325                        self.state.event(
7326                            "review",
7327                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7328                        );
7329                    }
7330                }
7331                records.push(record);
7332            }
7333
7334            // Tally the round's votes and, if they split, spend the one
7335            // round of reconsideration the split -> deliberate -> revote
7336            // shape `judge`/`vote` use for the panel, sized down to what a
7337            // read-only review round can afford: one round, and a revote
7338            // rather than an argument, because the panel already wrote its
7339            // reasoning down as findings the first time around.
7340            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7341            let vote_split =
7342                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7343            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7344            if vote_split {
7345                self.state.event(
7346                    "review",
7347                    format!(
7348                        "round {round}: votes split ({}) — one round of reconsideration",
7349                        initial_votes
7350                            .iter()
7351                            .map(|v| v.label())
7352                            .collect::<Vec<_>>()
7353                            .join(", ")
7354                    ),
7355                );
7356                // Seats read every seat's findings and votes, still numbered
7357                // and never named — the same anonymity `review` itself keeps.
7358                let panel: Vec<ReviewSeatReport<'_>> = records
7359                    .iter()
7360                    .filter_map(|r| {
7361                        r.vote.map(|vote| ReviewSeatReport {
7362                            reviewer: r.reviewer,
7363                            vote,
7364                            summary: &r.summary,
7365                            findings: &r.findings,
7366                        })
7367                    })
7368                    .collect();
7369
7370                let mut jobs = Vec::new();
7371                let mut seats_at = Vec::new();
7372                for (r, spec) in reviewers.iter().cloned().enumerate() {
7373                    // A seat with no initial vote has nothing to reconsider
7374                    // from and stays absent, the same as it stayed absent
7375                    // from `panel` above.
7376                    if records[r].vote.is_none() {
7377                        continue;
7378                    }
7379                    let wt = root.join(format!("review-{}", r + 1));
7380                    let seat_key = format!("review-{}", r + 1);
7381                    let spec = self.occupant(&seat_key, spec);
7382                    let seat = self.seat(&seat_key, &spec.id);
7383                    // A seat with no live session has already forgotten the
7384                    // initial review's prompt — restate the patch it is
7385                    // voting on, the same as `deliberate`/`vote` do for a
7386                    // judge in the same position.
7387                    // The panel already carries this seat's own review and
7388                    // vote, so restating the patch makes the prompt whole for
7389                    // a seat handed to another agent.
7390                    let build = |with_patch: bool| {
7391                        prompt::review_reconsider(&ReviewReconsiderCtx {
7392                            instruction: &self.state.instruction,
7393                            reviewer: r + 1,
7394                            lens: Lens::for_seat(r),
7395                            panel: &panel,
7396                            patch: with_patch.then_some(ReviewPatch {
7397                                branch: &winner.branch,
7398                                base_short: &base_short,
7399                                stat: &stat,
7400                                patch: &patch,
7401                            }),
7402                            round,
7403                            rounds: max_rounds,
7404                            language: &language,
7405                        })
7406                    };
7407                    let full = build(true);
7408                    let prompt = if has_context(&spec, &seat, sessions) {
7409                        build(false)
7410                    } else {
7411                        full.clone()
7412                    };
7413                    jobs.push(SeatJob {
7414                        prompt,
7415                        spec,
7416                        seat,
7417                        cwd: wt,
7418                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7419                        allow_write: false,
7420                        sessions,
7421                        artifacts: artifacts.clone(),
7422                        stem: format!("review-{round}-reconsider-{}", r + 1),
7423                        handover: Some(full),
7424                    });
7425                    seats_at.push(r);
7426                }
7427
7428                let mut recon_quota_losses = Vec::new();
7429                let recon_cache = self.state.config.cache_dir();
7430                let recon_ctx = WaveCtx {
7431                    carry_seats: true,
7432                    pauses: Some((&self.pause, &self.interrupt)),
7433                    run: &run_id,
7434                    node: "review",
7435                    prompts: &prompts,
7436                    cache: recon_cache.as_deref(),
7437                    round: Some(round),
7438                };
7439                let recon_results = ask_json_wave::<ReviewRevote>(
7440                    jobs,
7441                    Arc::clone(&self.sem),
7442                    review_retries,
7443                    &self.roles.reviewer_roster,
7444                    &recon_ctx,
7445                    &mut recon_quota_losses,
7446                    &mut self.state,
7447                    &|_: &ReviewRevote| Ok(()),
7448                )
7449                .await;
7450                if self.state.resource_block.is_some() {
7451                    return self.stop_for_resource();
7452                }
7453                self.state.quota.extend(recon_quota_losses);
7454
7455                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7456                    let agent_id = seat.agent.clone();
7457                    self.state.seats.insert(seat.key.clone(), seat);
7458                    let mut rec = ReviewRevoteRecord {
7459                        reviewer: r + 1,
7460                        agent: agent_id,
7461                        vote: None,
7462                        reason: String::new(),
7463                        failed: None,
7464                    };
7465                    match res {
7466                        Ok((rv, _)) => {
7467                            rec.vote = Some(rv.vote);
7468                            rec.reason =
7469                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7470                            self.state.event(
7471                                "review",
7472                                format!(
7473                                    "round {round}: reviewer {} revoted {}",
7474                                    r + 1,
7475                                    rv.vote.label()
7476                                ),
7477                            );
7478                        }
7479                        Err(e) => {
7480                            rec.failed = Some(e.to_string());
7481                            self.state.event(
7482                                "review",
7483                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7484                            );
7485                        }
7486                    }
7487                    reconsideration.push(rec);
7488                }
7489            } else if initial_votes.len() > 1 {
7490                self.state.event(
7491                    "review",
7492                    format!(
7493                        "round {round}: votes agreed ({}) — no reconsideration",
7494                        initial_votes[0].label()
7495                    ),
7496                );
7497            }
7498
7499            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7500            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7501            // A round that already has a blocking finding and a round left to
7502            // try is going back to the fixer no matter what `verify.e2e`
7503            // says, so running it first only spends the loop's slowest step
7504            // (minutes, for a Rust repo's full test suite) on a head about
7505            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7506            // runs once a round has no blocking findings left (see
7507            // `round_is_clean`, which a deferred — empty — `e2e` can never
7508            // satisfy since `blocking` is nonzero whenever this branch is
7509            // taken), and `stop_reviewing` forces a real run before it will
7510            // ever read a deferred round as green.
7511            let defer_e2e =
7512                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7513            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7514                let reason =
7515                    format!("{blocking} blocking finding(s) already required a fix this round");
7516                self.state.event(
7517                    "verify",
7518                    format!(
7519                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7520                         {}); it will run once a round has none left",
7521                        short(&head)
7522                    ),
7523                );
7524                (Vec::new(), false, true, Some(reason))
7525            } else {
7526                let e2e_commands = self.state.config.verify.e2e.clone();
7527                let cache_dir = self.state.config.cache_dir();
7528                let context = format!("round {round}");
7529                let wait = self.repository_wait();
7530                let (e2e, verify_retried) = with_cache_lease(
7531                    &mut self.state,
7532                    cache_dir.as_deref(),
7533                    "e2e",
7534                    "e2e",
7535                    &winner.worktree,
7536                    &head,
7537                    verify_timeout,
7538                    &context,
7539                    &wait,
7540                    |state, budget| {
7541                        let shell = shell.clone();
7542                        let e2e_commands = e2e_commands.clone();
7543                        let worktree = winner.worktree.clone();
7544                        let context = context.clone();
7545                        async move {
7546                            run_e2e_with_retry(
7547                                state,
7548                                &shell,
7549                                &e2e_commands,
7550                                &worktree,
7551                                budget,
7552                                &context,
7553                            )
7554                            .await
7555                        }
7556                    },
7557                )
7558                .await?;
7559                (e2e, verify_retried, false, None)
7560            };
7561
7562            let expected = records.len();
7563            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7564            let incomplete = answered < expected;
7565            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7566            let policy = self.state.config.graph.incomplete_review;
7567            let clean = round_is_clean(
7568                blocking,
7569                e2e_ok,
7570                answered,
7571                expected,
7572                round_quota_missing,
7573                policy,
7574            );
7575
7576            let mut round_record = ReviewRound {
7577                round,
7578                head: head.clone(),
7579                verified_head: None,
7580                verified_at: None,
7581                reviews: records,
7582                e2e,
7583                verify_retried,
7584                e2e_deferred,
7585                e2e_defer_reason,
7586                fix: None,
7587                blocking,
7588                answered,
7589                expected,
7590                clean,
7591                progressed: false,
7592                vote_split,
7593                reconsideration,
7594                verdict: None,
7595            };
7596            // The final vote per seat is its revote where reconsideration
7597            // ran and answered, its initial vote otherwise — the same
7598            // fallback `tally` uses for a judge whose private vote failed.
7599            round_record.verdict = ReviewVote::worst(
7600                round_record
7601                    .final_votes()
7602                    .into_iter()
7603                    .map(|(_, _, vote)| vote),
7604            );
7605            // Which commit and when magi actually attempted to check —
7606            // known the moment a command was dispatched against `head`,
7607            // whether or not it finished: a resource-blocked attempt still
7608            // targeted a specific commit at a specific time, and leaving
7609            // that unrecorded is exactly what made `verification_summary`
7610            // report a fresh attempt as "commit unknown ... recorded before
7611            // this was tracked", indistinguishable from a genuinely old,
7612            // untracked record. Only a deferred or unconfigured round never
7613            // ran at all and has nothing to record — see
7614            // `ReviewRound::verified_head`'s own doc.
7615            if !matches!(
7616                round_record.e2e_status(),
7617                E2eStatus::Deferred | E2eStatus::NotConfigured
7618            ) {
7619                round_record.verified_head = Some(head.clone());
7620                round_record.verified_at = Some(Timestamp::now());
7621            }
7622
7623            if incomplete {
7624                let missing: Vec<String> = round_record
7625                    .reviews
7626                    .iter()
7627                    .filter(|r| r.failed.is_some())
7628                    .map(|r| format!("review-{}", r.reviewer))
7629                    .collect();
7630                self.state.event(
7631                    "review",
7632                    format!(
7633                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7634                        missing.join(", ")
7635                    ),
7636                );
7637            }
7638
7639            if clean {
7640                self.state.event(
7641                    "review",
7642                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7643                        format!(
7644                            "round {round}: clean (warn policy, incomplete panel) — no \
7645                             blocking findings from the seats that answered, verification green"
7646                        )
7647                    } else if incomplete {
7648                        format!(
7649                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7650                             quorum) — no blocking findings from the seats that answered, \
7651                             verification green",
7652                            expected - answered
7653                        )
7654                    } else {
7655                        format!("round {round}: clean — no blocking findings, verification green")
7656                    },
7657                );
7658                self.state.reviews.push(round_record);
7659                self.state.status = RunStatus::Gating;
7660                self.state.save()?;
7661                return Ok(());
7662            }
7663
7664            // Nothing was raised and verification passed, but not every seat
7665            // answered and `round_is_clean` still refused to call it clean —
7666            // either a seat is missing for a reason other than its own quota
7667            // (a crash, a timeout, unparsable output — worth another try), or
7668            // every seat that could have answered lost its quota and nobody
7669            // is left to decide on: re-review rather than send the fixer
7670            // after a round with nothing to fix.
7671            if incomplete && blocking == 0 && e2e_ok {
7672                self.state.reviews.push(round_record);
7673                self.state.save()?;
7674                if round == max_rounds {
7675                    self.state.status = RunStatus::Blocked;
7676                    self.state.event(
7677                        "review",
7678                        format!(
7679                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7680                             refusing to call it clean",
7681                            expected - answered
7682                        ),
7683                    );
7684                    return Ok(());
7685                }
7686                continue;
7687            }
7688
7689            // Nothing for the fixer to act on (`blocking == 0`) and the only
7690            // reason this round is not clean is that magi itself never got
7691            // a command to run — the shared build cache, not the patch (see
7692            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7693            // the fixer would invite a change to appease contention that has
7694            // nothing to do with the diff, and would leave this attempt
7695            // sitting in the next round's prompt as if it were about an
7696            // earlier, superseded commit rather than what it actually is:
7697            // the same head, still waiting to be checked. Wait for it the
7698            // same way the final round's own contention is already handled,
7699            // whatever round this happens to be.
7700            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7701                self.state.reviews.push(round_record);
7702                return self
7703                    .stop_unresolved_review(
7704                        "the round's own verification could not run",
7705                        &shell,
7706                        &winner.worktree,
7707                    )
7708                    .await;
7709            }
7710
7711            if round == max_rounds {
7712                self.state.reviews.push(round_record);
7713                return self
7714                    .stop_reviewing(
7715                        &format!(
7716                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7717                        ),
7718                        &shell,
7719                        &winner.worktree,
7720                    )
7721                    .await;
7722            }
7723
7724            if !self.fix_review_round(&winner, round_record).await? {
7725                return Ok(());
7726            }
7727        }
7728        Ok(())
7729    }
7730
7731    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7732    /// False means admission or continuation policy stopped the review loop.
7733    async fn fix_review_round(
7734        &mut self,
7735        winner: &Candidate,
7736        mut round_record: ReviewRound,
7737    ) -> Result<bool> {
7738        let round = round_record.round;
7739        let shell = self.state.config.shell();
7740        let step = self.run_fix_round(winner, &mut round_record).await?;
7741        if matches!(step, FixStep::Resource) {
7742            self.stop_for_resource()?;
7743            return Ok(false);
7744        }
7745        if self.state.reviews.len() < round {
7746            self.state.reviews.push(round_record);
7747        } else {
7748            self.state.reviews[round - 1] = round_record;
7749        }
7750        self.state.save()?;
7751
7752        // The fixer's own report never came back this round, even after
7753        // `continue_fix_report`'s own budget was spent on it — not an
7754        // ordinary "no report" (dropped stream, quota, plain failure),
7755        // which already reads that way and is left to the existing round
7756        // budget. Stopping here, rather than opening another round, is
7757        // what keeps a next reviewer/fixer wave from ever being
7758        // dispatched onto `winner.worktree` while whatever the seat's
7759        // last call may still have running there is unaccounted for: no
7760        // process liveness check exists (and none is being added — see
7761        // AGENTS.md/this task's own scope), so the only way to honour
7762        // "nothing starts before a valid report returns" is to not start
7763        // anything further on this worktree from this run at all.
7764        if matches!(step, FixStep::Recorded { report_lost: true }) {
7765            self.stop_reviewing(
7766                "the fixer's adoption report never came back, even after resuming its \
7767                         own seat; refusing to start another round against the same worktree \
7768                         while that is unresolved",
7769                &shell,
7770                &winner.worktree,
7771            )
7772            .await?;
7773            return Ok(false);
7774        }
7775
7776        let streak = self
7777            .state
7778            .reviews
7779            .iter()
7780            .rev()
7781            .take_while(|r| !r.progressed)
7782            .count();
7783        if streak >= STAGNANT_LIMIT {
7784            self.stop_reviewing(
7785                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7786                &shell,
7787                &winner.worktree,
7788            )
7789            .await?;
7790            return Ok(false);
7791        }
7792        Ok(true)
7793    }
7794
7795    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7796    async fn run_fix_round(
7797        &mut self,
7798        winner: &Candidate,
7799        round_record: &mut ReviewRound,
7800    ) -> Result<FixStep> {
7801        let round = round_record.round;
7802        let max_rounds = self.state.config.graph.review_rounds;
7803        let run_id = self.state.id.clone();
7804        let prompts = self.state.config.prompts.clone();
7805        let language = self.state.config.graph.language.clone();
7806        let sessions = self.state.config.graph.sessions;
7807        let artifacts = self.merge_repair_artifacts();
7808        let base = self.landing_base();
7809        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7810        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7811        let this_round_verification = round_record.verification_summary(&head);
7812        let all_findings: Vec<_> = round_record
7813            .reviews
7814            .iter()
7815            .flat_map(|r| r.findings.iter().cloned())
7816            .collect();
7817        // Fix. The winner's own implementer seat continues its conversation:
7818        // the competition is over, so context is pure benefit now.
7819        let blocking_findings: Vec<_> = all_findings
7820            .iter()
7821            .filter(|f| f.severity.blocks())
7822            .cloned()
7823            .collect();
7824        let fix_prompt = prompt::fix_in(
7825            &self.state.instruction,
7826            &blocking_findings,
7827            this_round_verification.as_ref(),
7828            round,
7829            max_rounds,
7830            &language,
7831            &self.state.config.graph.github_language,
7832        );
7833        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7834        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7835        let (job, seat, out) = self
7836            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7837                prompt: fix_prompt.clone(),
7838                spec,
7839                seat,
7840                cwd: winner.worktree.clone(),
7841                timeout,
7842                allow_write: true,
7843                sessions,
7844                artifacts: artifacts.clone(),
7845                stem: format!("fix-{round}"),
7846                handover: None,
7847            })
7848            .await;
7849        // Not asked: the round is not recorded (so a resume repeats it),
7850        // no fixer was handed over to, and the seat is left as it was.
7851        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7852            return Ok(FixStep::Resource);
7853        }
7854        let agent_id = seat.agent.clone();
7855
7856        let mut fix = FixRecord {
7857            agent: agent_id,
7858            addressed: Vec::new(),
7859            rejected: Vec::new(),
7860            notes: String::new(),
7861            committed: false,
7862            failed: None,
7863            duration_ms: 0,
7864            continuation: None,
7865        };
7866        let mut continuation = ContinuationRecord::not_needed();
7867        let mut final_seat = seat.clone();
7868        match out {
7869            AgentOutcome::Ok(o) => {
7870                fix.duration_ms = o.duration_ms;
7871                let parsed = verdict::extract_json::<FixReport>(&o.text);
7872                // A parsed report standing next to a command this same
7873                // reply's own CLI never confirmed the exit status of is
7874                // not a resolved answer — the identical `CommandEvidence`
7875                // `state.jobs` renders, read here instead of only on
7876                // display, per the completion judgment and the shown
7877                // record needing to agree.
7878                let incomplete_reason = match &parsed {
7879                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7880                        "the reply parsed, but it reported a command whose own CLI never \
7881                             confirmed an exit status"
7882                            .to_owned(),
7883                    ),
7884                    Ok(_) => None,
7885                    Err(e) => Some(e.to_string()),
7886                };
7887                match incomplete_reason {
7888                    None => {
7889                        let report = parsed.expect("checked Ok above");
7890                        fix.addressed = report.addressed;
7891                        fix.rejected = report.rejected;
7892                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7893                    }
7894                    Some(reason) => {
7895                        let (resumed_seat, resolved, failure, cont) = self
7896                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7897                            .await;
7898                        fix.duration_ms += cont.cumulative_wait_ms;
7899                        continuation = cont;
7900                        final_seat = resumed_seat;
7901                        match resolved {
7902                            Some(report) => {
7903                                fix.addressed = report.addressed;
7904                                fix.rejected = report.rejected;
7905                                fix.notes =
7906                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7907                            }
7908                            None => fix.failed = failure,
7909                        }
7910                    }
7911                }
7912            }
7913            // The CLI's raw error JSON is not a fix report to parse.
7914            AgentOutcome::Dropped(o) => {
7915                fix.duration_ms = o.duration_ms;
7916                let why = o
7917                    .dropped
7918                    .as_ref()
7919                    .map(|d| d.why.as_str())
7920                    .unwrap_or("the CLI ended the stream without delivering its answer");
7921                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7922            }
7923            AgentOutcome::Quota(o) => {
7924                self.state.quota.push(QuotaLoss::from_report(
7925                    final_seat.key.clone(),
7926                    "fix",
7927                    o.quota.as_ref(),
7928                ));
7929                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7930            }
7931            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7932        }
7933        if self.state.resource_block.is_some() {
7934            self.state.seats.insert(final_seat.key.clone(), final_seat);
7935            return Ok(FixStep::Resource);
7936        }
7937        fix.continuation = Some(continuation);
7938        self.state.seats.insert(final_seat.key.clone(), final_seat);
7939        let coordination = self.coordinate(&winner.worktree).await?;
7940        if let Ok(r) = git::rescue_commit(
7941            &winner.worktree,
7942            &format!("magi: review round {round} fixes (uncommitted work)"),
7943        )
7944        .await
7945        {
7946            self.state.note_withheld("fix", &r.withheld);
7947        }
7948        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7949        fix.committed = after != before;
7950        // Judged by what `git` says moved against base, never by the
7951        // fixer's own `addressed`/`rejected` count — see
7952        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7953        // `patch` snapshot above: swallowing this error would default
7954        // `diff_after` to empty, which almost always differs from a
7955        // non-empty `patch` and reads as "progressed" — exactly backwards
7956        // for a `git` failure the stagnation check cannot see through.
7957        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7958        drop(coordination);
7959        let progressed = diff_after != patch;
7960        let commit_note = if fix.committed {
7961            "committed"
7962        } else {
7963            "NO new commit"
7964        };
7965        let tree_note = if progressed {
7966            "changed vs base"
7967        } else {
7968            "unchanged vs base"
7969        };
7970        self.state.event(
7971            "fix",
7972            match &fix.failed {
7973                // Distinct on purpose from "0 addressed, 0 rejected": the
7974                // fixer's own diff still landed (blocking counts do keep
7975                // falling round over round), only its adoption report did
7976                // not come back, so this must never read like every
7977                // finding was reviewed and declined.
7978                Some(reason) => {
7979                    format!(
7980                        "round {round}: fixer's adoption report was lost ({reason}); \
7981                             {commit_note}, tree {tree_note}"
7982                    )
7983                }
7984                None => format!(
7985                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7986                         {tree_note}{}",
7987                    fix.addressed.len(),
7988                    fix.rejected.len(),
7989                    if continuation.outcome == ContinuationOutcome::Resumed {
7990                        format!(
7991                            " (adoption report recovered after {} continuation(s))",
7992                            continuation.attempts
7993                        )
7994                    } else {
7995                        String::new()
7996                    },
7997                ),
7998            },
7999        );
8000        round_record.fix = Some(fix);
8001        round_record.progressed = progressed;
8002        Ok(FixStep::Recorded {
8003            report_lost: matches!(
8004                continuation.outcome,
8005                ContinuationOutcome::Exhausted
8006                    | ContinuationOutcome::QuotaLost
8007                    | ContinuationOutcome::NoSession
8008            ),
8009        })
8010    }
8011
8012    /// Decide, from the last recorded round's own verification, whether
8013    /// stopping the review loop is a hand-off or a genuine block.
8014    ///
8015    /// Called once the loop has given up trying — the round budget is spent,
8016    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8017    /// findings still open, never while a round is still clean or the
8018    /// incomplete-panel case handled inline above. Gate and e2e are facts
8019    /// about the tree; a lingering review finding is an opinion, and this
8020    /// workload's own `magi stats` puts reviewer precision low enough
8021    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8022    /// must not by itself stand between a green, verified change and the
8023    /// human who decides what to do with it. A red e2e is not an opinion, so
8024    /// that case still blocks, with the failing command and a tail of its
8025    /// output recorded here rather than left in `run.json` for someone to go
8026    /// find.
8027    ///
8028    /// A round that deferred its own e2e (see [`Config::graph`]'s
8029    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8030    /// only because nothing ran, and treating an empty list as a passing one
8031    /// here is exactly the "deferred painted green" bug this function exists
8032    /// to not have. When the last round's own verification never resolved —
8033    /// deferred on purpose, or a real attempt the shared build cache blocked
8034    /// — this makes (or retries) the real run, on the actual worktree this
8035    /// loop is about to stop touching, before deciding anything. A
8036    /// resource-blocked attempt is likewise never read as either green or
8037    /// red: it is evidence about the machine, not the patch (see
8038    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8039    /// blocked cache leaves this call without deciding rather than guessing
8040    /// — the caller retries on a later reentry.
8041    /// Record, once, that the review loop handed off over a blocking finding
8042    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8043    /// `land` asks the owner even with `land_approval` off. Called from every
8044    /// path that concludes `Gating`; a reentry keeps the first record.
8045    fn record_contested_handoff(&mut self) {
8046        if self.state.contested_handoff.is_some() {
8047            return;
8048        }
8049        let Some(contested) = self
8050            .state
8051            .reviews
8052            .last()
8053            .and_then(ReviewRound::contested_handoff)
8054        else {
8055            return;
8056        };
8057        self.state.event(
8058            "review",
8059            format!(
8060                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8061                 wait for the owner's approval",
8062                contested.findings.len(),
8063                contested.rejecters.len()
8064            ),
8065        );
8066        self.state.contested_handoff = Some(contested);
8067    }
8068
8069    async fn stop_unresolved_review(
8070        &mut self,
8071        why: &str,
8072        shell: &[String],
8073        worktree: &Path,
8074    ) -> Result<()> {
8075        self.stop_reviewing_inner(why, shell, worktree, true).await
8076    }
8077
8078    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8079        self.stop_reviewing_inner(why, shell, worktree, false).await
8080    }
8081
8082    async fn stop_reviewing_inner(
8083        &mut self,
8084        why: &str,
8085        shell: &[String],
8086        worktree: &Path,
8087        repairable: bool,
8088    ) -> Result<()> {
8089        let round_idx = self.state.reviews.len() - 1;
8090        // A deferred round and a resource-blocked one are the same shape
8091        // here: neither has a real result yet, and both get one more
8092        // attempt. Read off `e2e_status` — the single source for this —
8093        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8094        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8095        // still retries instead of being read as a settled result the
8096        // instant it stops being empty.
8097        let needs_catchup_run = matches!(
8098            self.state.reviews[round_idx].e2e_status(),
8099            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8100        );
8101        if needs_catchup_run {
8102            let round = self.state.reviews[round_idx].round;
8103            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8104            let commands = self.state.config.verify.e2e.clone();
8105            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8106            let cache_dir = self.state.config.cache_dir();
8107            let context = format!(
8108                "round {round}: verification unresolved, catching up before the final decision"
8109            );
8110            let wait = self.repository_wait();
8111            let (outcomes, verify_retried) = with_cache_lease(
8112                &mut self.state,
8113                cache_dir.as_deref(),
8114                "e2e",
8115                "e2e",
8116                worktree,
8117                &attempted_head,
8118                timeout,
8119                &context,
8120                &wait,
8121                |state, budget| {
8122                    let shell = shell.to_vec();
8123                    let commands = commands.clone();
8124                    let context = context.clone();
8125                    async move {
8126                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8127                            .await
8128                    }
8129                },
8130            )
8131            .await?;
8132            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8133                self.state.record_base_progress(&attempted_head);
8134            }
8135            let last = &mut self.state.reviews[round_idx];
8136            last.e2e = outcomes;
8137            last.verify_retried = verify_retried;
8138            // Always the commit and time this attempt actually targeted,
8139            // whether or not it happens to equal the reviewed `head` and
8140            // whether or not a command finished — see
8141            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8142            // attempt is recorded too, so a later reader sees "attempted
8143            // again at T2" rather than silence.
8144            last.verified_head = Some(attempted_head);
8145            last.verified_at = Some(Timestamp::now());
8146            if verify_inconclusive(&last.e2e) {
8147                // Still not a real result: `e2e_deferred` is left exactly
8148                // as it was, so `needs_catchup_run` above reads
8149                // `ResourceBlocked` (via `e2e_status`, which checks
8150                // `resource_blocked` before `e2e_deferred`) and retries
8151                // again on the next reentry, rather than recording
8152                // contention as a red e2e and blocking the run on it.
8153                self.state.save()?;
8154                return Ok(());
8155            }
8156            last.e2e_deferred = false;
8157            // A complete panel may have been saved as unclean solely because
8158            // verification could not run. Refresh that verdict with the new
8159            // outcomes before a granted repair consumes the round's result.
8160            // Incomplete panels retain their original policy/quota decision.
8161            if last.answered == last.expected {
8162                last.clean = round_is_clean(
8163                    last.blocking,
8164                    last.e2e.iter().all(CommandOutcome::ok),
8165                    last.answered,
8166                    last.expected,
8167                    0,
8168                    self.state.config.graph.incomplete_review,
8169                );
8170            }
8171        }
8172        let last = &self.state.reviews[round_idx];
8173        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8174
8175        match last.e2e_status() {
8176            E2eStatus::Failed => {
8177                let red: Vec<String> = last
8178                    .e2e
8179                    .iter()
8180                    .filter(|o| !o.ok())
8181                    .map(|o| {
8182                        format!(
8183                            "`{}` -> {:?}\n{}",
8184                            o.command,
8185                            o.code,
8186                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8187                        )
8188                    })
8189                    .collect();
8190                let can_repair = last.blocking == 0 && last.fix.is_none();
8191                self.state.reviews[round_idx].clean = false;
8192                self.state
8193                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8194                self.state.status = RunStatus::Blocked;
8195                if repairable
8196                    && can_repair
8197                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8198                {
8199                    self.state.status = RunStatus::Reviewing;
8200                }
8201            }
8202            // `needs_catchup_run` above already retried once this call; if
8203            // it is still blocked, this is magi's own admission it could
8204            // not get a command to run, never a verdict on the patch — the
8205            // run is left exactly where a later reentry can retry again.
8206            E2eStatus::ResourceBlocked => {
8207                self.state.event(
8208                    "review",
8209                    format!(
8210                        "{why}; e2e could not run (resource unavailable); not \
8211                         deciding yet"
8212                    ),
8213                );
8214            }
8215            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8216                self.state.event(
8217                    "review",
8218                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8219                );
8220                self.record_contested_handoff();
8221                self.state.status = RunStatus::Gating;
8222            }
8223        }
8224        self.state.save()?;
8225        Ok(())
8226    }
8227
8228    // ----------------------------------------------------------------- gate
8229
8230    async fn gate(&mut self) -> Result<()> {
8231        if self.hold_base_sync().await? {
8232            return Ok(());
8233        }
8234        // Judged by the review record itself, not by `status`: a solo
8235        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8236        // reentry (see `judge`), and trusting it here is exactly how a run
8237        // that exhausted its review budget got gated and merged a second
8238        // time around. `review_conclusion` recomputes the review loop's own
8239        // verdict from the round records themselves — `Gating` for a clean
8240        // round or a hand-off (see `stop_reviewing`), anything else means the
8241        // loop is still going or genuinely blocked.
8242        // A base the winner could not be replayed onto is a decision, not a
8243        // round: there is no landing tree to gate. Read as its own record for
8244        // the same reason the review verdict is.
8245        if self.state.status == RunStatus::Failed
8246            || self
8247                .state
8248                .base_sync
8249                .as_ref()
8250                .is_some_and(|s| s.conflict.is_some())
8251            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8252                != Some(RunStatus::Gating)
8253        {
8254            return Ok(());
8255        }
8256        let Some(winner) = self.state.winner().cloned() else {
8257            return Ok(());
8258        };
8259        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8260        self.state.reconcile_verification_head(&head);
8261        self.state.save()?;
8262        let wait = self.repository_wait();
8263        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8264            return Ok(());
8265        }
8266        if self.state.gate_ran {
8267            // `review_loop` derives its conclusion from the clean review
8268            // record on every reentry and therefore puts a completed run back
8269            // in `Gating`. A recorded gate is a stronger, terminal fact:
8270            // retain its original command output (or lack of any, for a repo
8271            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8272            // doc) and restore `Blocked` on a real failure rather than
8273            // pretending the command is still running or running it a second
8274            // time. `gate_ran == false` remains the only shape — unattempted,
8275            // or a resource-blocked retry — that may still need to execute a
8276            // command.
8277            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8278                self.state.status = RunStatus::Blocked;
8279                self.state.save()?;
8280            }
8281            return Ok(());
8282        }
8283        self.state.status = RunStatus::Gating;
8284        let mut outcomes = self.run_gate(&winner).await?;
8285        let mut attempted_head = self.state.gate_head.clone();
8286        loop {
8287            // A resource-blocked outcome means the gate command never actually
8288            // ran - the shared build cache could not be acquired or confirmed
8289            // fresh in time - which is evidence about the machine, not about
8290            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8291            // Recording it as a red gate would mark a run `Blocked` on nothing
8292            // but contention magi has already logged; leaving `self.state.gate`
8293            // empty and `self.state.gate_ran` false instead keeps the shape
8294            // this function already treats as "still needs to run" (see the
8295            // early-return above), so the next call retries the command
8296            // rather than concluding anything.
8297            if verify_inconclusive(&outcomes) {
8298                self.state.save()?;
8299                return Ok(());
8300            }
8301            if outcomes.iter().all(CommandOutcome::ok) {
8302                break;
8303            }
8304            let fixes_before = self.state.gate_fixes.len();
8305            match self.gate_fix_round(&winner, &outcomes).await? {
8306                GateFix::Retry => {
8307                    outcomes = self.run_gate(&winner).await?;
8308                    attempted_head = self.state.gate_head.clone();
8309                }
8310                GateFix::Stop => {
8311                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8312                    if self.state.gate_fixes.len() > fixes_before
8313                        && attempted_head.as_deref() != Some(current.as_str())
8314                        && verify_current_head(&mut self.state, &winner, &wait).await?
8315                    {
8316                        // A new SHA owes a gate even when a spent fixer changed only
8317                        // commit metadata. This does not grant another repair round.
8318                        outcomes = self.run_gate(&winner).await?;
8319                        attempted_head = self.state.gate_head.clone();
8320                    } else {
8321                        break;
8322                    }
8323                }
8324                GateFix::Defer => {
8325                    self.state.save()?;
8326                    return Ok(());
8327                }
8328            }
8329        }
8330        let passed = outcomes.iter().all(CommandOutcome::ok);
8331        self.state.gate = outcomes;
8332        self.state.gate_ran = true;
8333        self.state.gate_head = attempted_head;
8334        if !self.state.gate.is_empty()
8335            && !verify_inconclusive(&self.state.gate)
8336            && let Some(head) = self.state.gate_head.clone()
8337        {
8338            self.state.record_base_progress(&head);
8339        }
8340        self.state.save()?;
8341        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8342            return Ok(());
8343        }
8344        if !passed {
8345            self.state.status = RunStatus::Blocked;
8346            let spent = self.state.gate_fixes.len();
8347            self.state.event(
8348                "gate",
8349                if spent == 0 {
8350                    "gate failed; not merging".to_owned()
8351                } else {
8352                    format!("gate failed after {spent} gate-fix round(s); not merging")
8353                },
8354            );
8355        }
8356        self.state.save()?;
8357        Ok(())
8358    }
8359
8360    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8361    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8362    /// after review is clean and never on a candidate awaiting judging.
8363    ///
8364    /// Never fails the run: a non-zero exit or timeout is a warning and a
8365    /// recorded outcome, and the gate remains the single arbiter. Nothing
8366    /// configured means nothing happens - no event, no commit. `commit_all`
8367    /// commits any leftover change under the neutral identity and returns
8368    /// `false` when the tree is clean, so no empty commit is ever made.
8369    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8370        let commands = state.config.verify.pre_gate.clone();
8371        if commands.is_empty() {
8372            return;
8373        }
8374        let shell = state.config.shell();
8375        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8376        let (outcomes, _) = run_commands(
8377            state,
8378            "pre_gate",
8379            "pre_gate",
8380            0,
8381            &shell,
8382            &commands,
8383            &winner.worktree,
8384            timeout,
8385        )
8386        .await;
8387        for o in &outcomes {
8388            if !o.ok() {
8389                tracing::warn!(
8390                    "pre_gate `{}` failed ({:?}); the gate decides",
8391                    o.command,
8392                    o.code
8393                );
8394            }
8395            state.event(
8396                "pre_gate",
8397                format!(
8398                    "`{}` -> {}",
8399                    o.command,
8400                    if o.ok() {
8401                        "pass".to_owned()
8402                    } else {
8403                        format!(
8404                            "FAIL ({:?})\n{}",
8405                            o.code,
8406                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8407                        )
8408                    }
8409                ),
8410            );
8411        }
8412        state.pre_gate = outcomes;
8413        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8414            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8415                Ok(head) => {
8416                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8417                    state.pre_gate_commit = Some(head);
8418                }
8419                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8420            },
8421            Ok(false) => {}
8422            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8423        }
8424        if let Err(e) = state.save() {
8425            tracing::warn!("could not persist the pre_gate record: {e:#}");
8426        }
8427    }
8428
8429    /// Run `verify.gate` once against the winner's current tree, logging one
8430    /// event per command. Empty when nothing is configured.
8431    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8432        let wait = self.repository_wait();
8433        run_landing_gate(&mut self.state, winner, &wait).await
8434    }
8435
8436    /// One bounded fix round for a failing gate.
8437    ///
8438    /// The fixer is told the failure came from the gate itself, not from a
8439    /// reviewer, and is shown the failed commands, their exit codes and a tail
8440    /// of their output - whatever `[verify].gate` holds, nothing here knows
8441    /// what those commands run. Only a normal non-zero exit that printed
8442    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8443    /// command or a full disk says nothing about the code, and a fixer sent
8444    /// after it can only appease the machine. The round is judged by what git
8445    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8446    /// again before the gate does, so a fix cannot trade a green gate for a
8447    /// red e2e unnoticed.
8448    async fn gate_fix_round(
8449        &mut self,
8450        winner: &Candidate,
8451        outcomes: &[CommandOutcome],
8452    ) -> Result<GateFix> {
8453        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8454            g.phase == crate::repair_budget::Phase::Fixing
8455                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8456        }) {
8457            self.state.gate_fixes.len().saturating_add(1)
8458        } else {
8459            self.state.config.graph.gate_fix_rounds
8460        };
8461        let spent = self.state.gate_fixes.len();
8462        if spent >= cap {
8463            if cap > 0 {
8464                self.state.event(
8465                    "gate",
8466                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8467                );
8468            }
8469            return Ok(GateFix::Stop);
8470        }
8471        if !gate_fixable(outcomes) {
8472            self.state.event(
8473                "gate",
8474                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8475                 command or similar); not spending a fix round on it",
8476            );
8477            return Ok(GateFix::Stop);
8478        }
8479        let min_free = self.state.config.disk.min_free_bytes;
8480        if min_free > 0 {
8481            match crate::disk::free_bytes(&winner.worktree) {
8482                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8483                Ok(free) => {
8484                    self.state.event(
8485                        "gate",
8486                        format!(
8487                            "only {free} bytes free ({min_free} required by `[disk] \
8488                             min_free_bytes`); not spending a fix round on a failure the disk \
8489                             may explain"
8490                        ),
8491                    );
8492                    return Ok(GateFix::Stop);
8493                }
8494                Err(e) => {
8495                    self.state.event(
8496                        "gate",
8497                        format!("free disk space could not be measured ({e:#}); no fix round"),
8498                    );
8499                    return Ok(GateFix::Stop);
8500                }
8501            }
8502        }
8503
8504        let attempt = spent + 1;
8505        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8506        let base = self.landing_base();
8507        let fix_prompt = prompt::gate_fix_in(
8508            &self.state.instruction,
8509            &failed,
8510            attempt,
8511            cap,
8512            &self.state.config.graph.language,
8513            &self.state.config.graph.github_language,
8514        );
8515        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8516        let sessions = self.state.config.graph.sessions;
8517        let artifacts = self.merge_repair_artifacts();
8518        self.state.event(
8519            "gate",
8520            format!("gate failed; gate-fix round {attempt} of {cap}"),
8521        );
8522        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8523        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8524        // Persist before invoking the fixer: it may commit and then be
8525        // interrupted before its result (or the following e2e) is saved.
8526        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8527            head: before.clone(),
8528            passed: false,
8529            outcomes: Vec::new(),
8530        });
8531        self.state.save()?;
8532        let (_, seat, out) = self
8533            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8534                prompt: fix_prompt.clone(),
8535                spec,
8536                seat,
8537                cwd: winner.worktree.clone(),
8538                timeout,
8539                allow_write: true,
8540                sessions,
8541                artifacts: artifacts.clone(),
8542                stem: format!("gate-fix-{attempt}"),
8543                handover: None,
8544            })
8545            .await;
8546        // The fixer was never asked: no round is spent (nothing is recorded)
8547        // and a later reentry asks the same agent again.
8548        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8549            return Ok(GateFix::Defer);
8550        }
8551        let mut record = GateFixRecord {
8552            agent: seat.agent.clone(),
8553            failed,
8554            notes: String::new(),
8555            committed: false,
8556            error: None,
8557        };
8558        match out {
8559            AgentOutcome::Ok(o) => {
8560                // A missing report is not a failed fix: the round is judged
8561                // by the tree below, and the report only carries prose.
8562                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8563                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8564                }
8565            }
8566            AgentOutcome::Dropped(_) => {
8567                record.error = Some("the CLI dropped the stream".to_owned());
8568            }
8569            AgentOutcome::Quota(o) => {
8570                self.state.quota.push(QuotaLoss::from_report(
8571                    seat.key.clone(),
8572                    "gate-fix",
8573                    o.quota.as_ref(),
8574                ));
8575                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8576            }
8577            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8578        }
8579        self.state.seats.insert(seat.key.clone(), seat);
8580        let coordination = self.coordinate(&winner.worktree).await?;
8581        if let Ok(r) = git::rescue_commit(
8582            &winner.worktree,
8583            &format!("magi: gate fix {attempt} (uncommitted work)"),
8584        )
8585        .await
8586        {
8587            self.state.note_withheld("gate-fix", &r.withheld);
8588        }
8589        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8590        record.committed = after != before;
8591        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8592        let note = record.error.clone();
8593        self.state.gate_fixes.push(record);
8594        self.state.save()?;
8595        if !changed {
8596            self.state.event(
8597                "gate",
8598                match note {
8599                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8600                    None => format!("gate-fix round {attempt}: the tree did not change"),
8601                },
8602            );
8603            return Ok(GateFix::Stop);
8604        }
8605        self.state.event(
8606            "gate",
8607            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8608        );
8609
8610        drop(coordination);
8611        let wait = self.repository_wait();
8612        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8613    }
8614
8615    // ---------------------------------------------------------------- merge
8616
8617    /// One read-only rewrite by the summary's author; a text the gate still
8618    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8619    /// applies on their word or on silence. `None` means the run is parked on
8620    /// that question and nothing may be pushed or opened yet.
8621    async fn guarded_pr_message(
8622        &mut self,
8623        winner: &Candidate,
8624        facts: Option<&BranchFacts>,
8625        posting: bool,
8626    ) -> Result<Option<PrMessage>> {
8627        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8628        let head = match final_head.as_deref() {
8629            Some(h) => crate::pr_outcome::FinalHead::At(h),
8630            None => crate::pr_outcome::FinalHead::Unknown,
8631        };
8632        // Only a pull request that is about to be posted pays for a summary.
8633        let summarized = if posting {
8634            Some(self.summarize_change(winner, final_head.as_deref()).await)
8635        } else {
8636            None
8637        };
8638        // A busy shared resource is not a permanent cause: publishing an
8639        // "unavailable" description would stick (a later head move never
8640        // edits the body). Stop before anything is pushed or opened; the
8641        // run is `Blocked` and a resume asks the summary again.
8642        if self.state.resource_block.is_some() {
8643            self.state.event(
8644                "pr-summary",
8645                "a shared resource was busy; nothing was published, resume to summarize again",
8646            );
8647            return Ok(None);
8648        }
8649        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8650            anyhow::bail!(
8651                "the final commit moved during public description generation; retry before publishing"
8652            );
8653        }
8654        let note = match (&summarized, &self.state.change_summary) {
8655            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8656            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8657            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8658            (None, _) => ChangeNote::NotAsked,
8659        };
8660        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8661        if !posting {
8662            let identity = crate::scrub::Identity::current();
8663            return Ok(Some(PrMessage {
8664                title: crate::scrub::scrub(&pr.title, &identity),
8665                body: crate::scrub::scrub(&pr.body, &identity),
8666            }));
8667        }
8668        // This very text was already rejected and asked about: no second
8669        // judge call, rewrite or question, just carry the record on.
8670        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8671        if let Some(known) = self
8672            .state
8673            .github_text
8674            .clone()
8675            .filter(|g| g.fingerprint == raw_fp)
8676        {
8677            return self.settle_withheld(winner, &pr, known).await;
8678        }
8679        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8680        let mut violations =
8681            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8682        let identity = crate::scrub::Identity::current();
8683        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8684            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8685            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8686        {
8687            violations.push(crate::github_text::Violation::SensitiveData);
8688        }
8689        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8690        // only a language violation starts one; sensitive data alone goes
8691        // straight to the owner, with the redacted text as the fallback.
8692        let has_language = violations
8693            .iter()
8694            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8695        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8696            self.state.event(
8697                "github-text",
8698                if has_language {
8699                    format!("description rejected: {violations:?}; requesting one rewrite")
8700                } else {
8701                    format!("description rejected: {violations:?}; asking the owner")
8702                },
8703            );
8704            let mut rewritten = false;
8705            let mut rejected_rewrite = None;
8706            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8707            {
8708                let output = text;
8709                if let Some(title) = summary_title(&output) {
8710                    // Judge the redacted candidate that will actually be posted.
8711                    // Sensitive data alone must not discard a useful rewrite.
8712                    let raw_body = summary_without_title(&output);
8713                    let clean_title = crate::scrub::scrub(&title, &identity);
8714                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8715                    let redacted = clean_title != title || body != raw_body;
8716                    let title = clean_title;
8717                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8718                    if !complete {
8719                        self.state.event(
8720                            "github-text",
8721                            "description rewrite omitted required findings",
8722                        );
8723                    }
8724                    if !body.trim().is_empty() {
8725                        body.push_str("\n\n");
8726                        body.push_str(&public_pr_evidence(&self.state, head));
8727                    }
8728                    let marker = format!("magi:run/{}", self.state.id);
8729                    if !body.trim().is_empty() && !body.contains(&marker) {
8730                        body.push_str(&format!("\n\n{marker}"));
8731                    }
8732                    let rewrite = PrMessage {
8733                        title: title.clone(),
8734                        body: body.clone(),
8735                    };
8736                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8737                    let mut failed = crate::github_text::check_for(
8738                        &self.state.config.graph,
8739                        &title,
8740                        &body,
8741                        redecision,
8742                    );
8743                    if (!crate::github_text::shareable(&title)
8744                        || !crate::github_text::shareable(&body))
8745                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8746                    {
8747                        failed.push(crate::github_text::Violation::SensitiveData);
8748                    }
8749                    if complete
8750                        && !title.trim().is_empty()
8751                        && !body.trim().is_empty()
8752                        && !failed.is_empty()
8753                    {
8754                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8755                            title: title.clone(),
8756                            body: body.clone(),
8757                            failed: failed
8758                                .iter()
8759                                .map(|v| crate::github_text::category(*v).to_owned())
8760                                .collect(),
8761                        });
8762                    }
8763                    if complete
8764                        && !title.trim().is_empty()
8765                        && !raw_body.trim().is_empty()
8766                        && failed.is_empty()
8767                    {
8768                        decision = redecision;
8769                        pr = PrMessage { title, body };
8770                        rewritten = true;
8771                        self.state
8772                            .event("github-text", "description rewrite passed");
8773                        if redacted {
8774                            self.state.event(
8775                                "github-text",
8776                                "sensitive data removed from description rewrite",
8777                            );
8778                        }
8779                    }
8780                }
8781            }
8782            if !rewritten {
8783                self.state.event(
8784                    "github-text",
8785                    "description rewrite unavailable or rejected; asking the owner",
8786                );
8787                if let Some(w) =
8788                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8789                {
8790                    // A different text than the one asked about before: that
8791                    // question no longer describes anything.
8792                    self.retire_github_text_question("the withheld text changed");
8793                    let record = crate::run::GithubTextAsk {
8794                        rewrite: rejected_rewrite,
8795                        fingerprint: raw_fp,
8796                        title: w.title,
8797                        body: w.body,
8798                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8799                        question: None,
8800                        asks: 0,
8801                        resolved: false,
8802                        chosen_title: None,
8803                        choice: None,
8804                    };
8805                    return self.settle_withheld(winner, &pr, record).await;
8806                }
8807            }
8808        }
8809        let (title, body) =
8810            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8811        Ok(Some(PrMessage { title, body }))
8812    }
8813
8814    /// Describe the whole change from its final diff in either run mode.
8815    ///
8816    /// The summary is tied to the final commit and to the merge base of the
8817    /// resolved remote base; a record for any other pair is dropped and the
8818    /// agent asked again, so a description never outlives the diff it was
8819    /// written from. The seat is read-only and the prompt tells it to read the
8820    /// diff and the code around it. `[roles] synthesizer` selects the agents
8821    /// (unset: the default pick), each asked at most once, advancing on an
8822    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8823    /// carries the reason it is unavailable; nothing is invented then.
8824    async fn summarize_change(
8825        &mut self,
8826        winner: &Candidate,
8827        head: Option<&str>,
8828    ) -> std::result::Result<(), String> {
8829        let Some(head) = head else {
8830            return Err("the final commit could not be read".to_owned());
8831        };
8832        let remote = self.state.config.merge.remote.clone();
8833        let tracking = format!("{remote}/{}", self.state.base_branch);
8834        if !git::rev_exists(&winner.worktree, &tracking).await {
8835            return Err(format!("the remote base `{tracking}` could not be read"));
8836        }
8837        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8838            Ok(b) if !b.is_empty() => b,
8839            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8840        };
8841        let evidence = crate::pr_outcome::summary_evidence(
8842            &self.state,
8843            crate::pr_outcome::FinalHead::At(head),
8844        );
8845        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8846        if self
8847            .state
8848            .change_summary
8849            .as_ref()
8850            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8851        {
8852            return Ok(());
8853        }
8854        if self.state.change_summary.take().is_some() {
8855            self.state.event(
8856                "pr-summary",
8857                "the final diff or evidence changed; the earlier change summary was discarded",
8858            );
8859        }
8860        // Save the source snapshot even if no generation agent is available.
8861        if let Err(e) = write_artifact(
8862            &self.state,
8863            &format!("public-description-{fingerprint}.md"),
8864            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8865        ) {
8866            tracing::warn!("could not save public description evidence: {e:#}");
8867        }
8868        let chain = match agent::pick_chain(
8869            &self.state.config.agents,
8870            self.state.config.roles.synthesizer.as_ref(),
8871            &agent::installed,
8872            "synthesizer",
8873        ) {
8874            Ok(chain) => chain,
8875            Err(e) => {
8876                tracing::warn!("no agent for the change summary: {e:#}");
8877                self.state.event("pr-summary", "no summary agent available");
8878                return Err("no agent was available to summarize it".to_owned());
8879            }
8880        };
8881        let subjects = git::subjects(&winner.worktree, &base, head)
8882            .await
8883            .unwrap_or_default();
8884        let stat = git::diff_stat(&winner.worktree, &base, head)
8885            .await
8886            .unwrap_or_default();
8887        let stat: String = stat.chars().take(6_000).collect();
8888        let mut prompt = crate::prompt::change_summary_in(
8889            &base,
8890            head,
8891            &subjects,
8892            stat.trim(),
8893            &self.state.config.graph.github_language,
8894        );
8895        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8896        let sessions = self.state.config.graph.sessions;
8897        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8898        let prompts = self.state.config.prompts.clone();
8899        let cache = self.state.config.cache_dir();
8900        let run = self.state.id.clone();
8901        for (i, spec) in chain.into_iter().enumerate() {
8902            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8903            let github_spec = match agent::for_github_text(
8904                &spec,
8905                &self.state.config.graph.github_language,
8906                &winner.worktree,
8907            ) {
8908                Ok(spec) => spec,
8909                Err(e) => {
8910                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8911                    continue;
8912                }
8913            };
8914            let job = SeatJob {
8915                spec: github_spec,
8916                seat,
8917                cwd: winner.worktree.clone(),
8918                prompt: prompt.clone(),
8919                timeout,
8920                allow_write: false,
8921                sessions,
8922                artifacts: agent::artifacts_dir(&self.state.dir()),
8923                stem: if i == 0 {
8924                    "change-summary".to_owned()
8925                } else {
8926                    format!("change-summary-{}", spec.id)
8927                },
8928                handover: None,
8929            };
8930            let ctx = WaveCtx {
8931                run: &run,
8932                node: "pr-summary",
8933                prompts: &prompts,
8934                cache: cache.as_deref(),
8935                round: None,
8936                carry_seats: false,
8937                pauses: Some((&self.pause, &self.interrupt)),
8938            };
8939            let (seat, outcome) =
8940                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8941            self.state.seats.insert(seat.key.clone(), seat);
8942            let text = match outcome {
8943                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8944                AgentOutcome::ResourceBlocked(why) => {
8945                    // No agent was asked: the run stops resumably before
8946                    // anything is published (see `guarded_pr_message`).
8947                    if self.state.resource_block.is_none() {
8948                        self.state.resource_block = Some(crate::run::ResourceBlock {
8949                            cause: crate::run::ResourceBlockCause::Other,
8950                            node: "pr-summary".to_owned(),
8951                            kind: "shared resource".to_owned(),
8952                            detail: why,
8953                            at: Timestamp::now(),
8954                        });
8955                    }
8956                    return Err("a shared resource was busy".to_owned());
8957                }
8958                _ => continue,
8959            };
8960            let blind = &self.state.config.blind;
8961            let Some(title) = summary_title(&text) else {
8962                continue;
8963            };
8964            let body: String = summary_without_title(&text)
8965                .lines()
8966                .filter(|l| !l.contains("magi:run/"))
8967                .collect::<Vec<_>>()
8968                .join("\n");
8969            let body = blind::sanitize_prose(body.trim(), blind);
8970            let title = blind::sanitize_prose(&title, blind);
8971            if body.trim().is_empty() || title.trim().is_empty() {
8972                continue;
8973            }
8974            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8975                continue;
8976            }
8977            // The branch must still be the one the agent was shown.
8978            if git::rev_parse(&winner.worktree, "HEAD")
8979                .await
8980                .ok()
8981                .as_deref()
8982                != Some(head)
8983            {
8984                return Err("the branch moved while the summary was written".to_owned());
8985            }
8986            if git::merge_base(&winner.worktree, &tracking, head)
8987                .await
8988                .ok()
8989                .as_deref()
8990                != Some(base.as_str())
8991                || crate::pr_outcome::summary_evidence(
8992                    &self.state,
8993                    crate::pr_outcome::FinalHead::At(head),
8994                ) != evidence
8995            {
8996                return Err("the summary evidence changed while it was written".to_owned());
8997            }
8998            self.state.change_summary = Some(crate::run::ChangeSummary {
8999                head: head.to_owned(),
9000                base,
9001                evidence: fingerprint,
9002                title,
9003                body,
9004            });
9005            self.state.event(
9006                "pr-summary",
9007                format!("change summary written by {} for the final diff", spec.id),
9008            );
9009            if let Err(e) = self.state.save() {
9010                tracing::warn!("could not save the change summary: {e:#}");
9011            }
9012            return Ok(());
9013        }
9014        self.state
9015            .event("pr-summary", "no agent produced a change summary");
9016        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9017    }
9018
9019    /// One read-only rewrite request for a withheld description. The
9020    /// implementer's own conversation is tried first when it exists and can be
9021    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9022    /// writes it in a separate conversation, skipping the implementer's agent.
9023    /// The prompt carries the whole text, so no agent needs any memory. Each
9024    /// agent is asked at most once and the chain advances on an error, quota,
9025    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9026    /// that has run out of quota must not end the rewrite. Nothing is recorded
9027    /// as a handover.
9028    async fn rewrite_pr_text(
9029        &mut self,
9030        winner: &Candidate,
9031        pr: &PrMessage,
9032        violations: &[crate::github_text::Violation],
9033    ) -> Option<String> {
9034        let prompt = format!(
9035            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9036            pr.title, pr.body
9037        );
9038        let sessions = self.state.config.graph.sessions;
9039        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9040        // (spec, the resumed implementer's seat). A fallback seat is created
9041        // only when its turn comes, so an unused fallback leaves no record.
9042        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9043        let key = format!("impl-{}", winner.label);
9044        if let Some(spec) = self
9045            .state
9046            .config
9047            .agents
9048            .iter()
9049            .find(|a| a.id == winner.agent)
9050            .cloned()
9051            && let Some(existing) = self.state.seats.get(&key)
9052            && existing.agent == spec.id
9053            && agent::has_session(spec.kind, existing, sessions)
9054        {
9055            attempts.push((spec, Some(existing.clone())));
9056        }
9057        match agent::pick_chain(
9058            &self.state.config.agents,
9059            self.state.config.roles.language_judge.as_ref(),
9060            &agent::installed,
9061            "language_judge",
9062        ) {
9063            Ok(chain) => {
9064                for spec in chain {
9065                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9066                        continue;
9067                    }
9068                    attempts.push((spec, None));
9069                }
9070            }
9071            Err(e) if attempts.is_empty() => {
9072                tracing::warn!("no agent for the description rewrite: {e:#}");
9073                self.state
9074                    .event("github-text", "no rewrite agent available");
9075                return None;
9076            }
9077            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9078        }
9079        let prompts = self.state.config.prompts.clone();
9080        let cache = self.state.config.cache_dir();
9081        let run = self.state.id.clone();
9082        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9083            let resumed = resumed_seat.is_some();
9084            let seat = resumed_seat.unwrap_or_else(|| {
9085                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9086            });
9087            let github_spec = match agent::for_github_text(
9088                &spec,
9089                &self.state.config.graph.github_language,
9090                &winner.worktree,
9091            ) {
9092                Ok(spec) => spec,
9093                Err(e) => {
9094                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9095                    continue;
9096                }
9097            };
9098            let job = SeatJob {
9099                spec: github_spec,
9100                seat,
9101                cwd: winner.worktree.clone(),
9102                prompt: prompt.clone(),
9103                timeout: retry_budget(timeout_full, resumed),
9104                allow_write: false,
9105                sessions,
9106                artifacts: agent::artifacts_dir(&self.state.dir()),
9107                stem: if i == 0 {
9108                    "github-text-rewrite".to_owned()
9109                } else {
9110                    format!("github-text-rewrite-{}", spec.id)
9111                },
9112                handover: None,
9113            };
9114            let ctx = WaveCtx {
9115                run: &run,
9116                node: "github-text",
9117                prompts: &prompts,
9118                cache: cache.as_deref(),
9119                round: None,
9120                carry_seats: false,
9121                pauses: Some((&self.pause, &self.interrupt)),
9122            };
9123            let (seat, outcome) =
9124                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9125            self.state.seats.insert(seat.key.clone(), seat);
9126            match outcome {
9127                AgentOutcome::Ok(o)
9128                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9129                {
9130                    return Some(o.text);
9131                }
9132                AgentOutcome::ResourceBlocked(_) => return None,
9133                _ => {}
9134            }
9135        }
9136        None
9137    }
9138
9139    /// Abandon the standing posting-gate question, if any.
9140    fn retire_github_text_question(&mut self, why: &str) {
9141        let Some(id) = self
9142            .state
9143            .github_text
9144            .as_ref()
9145            .and_then(|g| g.question.clone())
9146        else {
9147            return;
9148        };
9149        let _ = ask::Questions::open().update(&id, |q| {
9150            q.abandon(why);
9151            Ok(())
9152        });
9153    }
9154
9155    /// Carry a withheld text to its end: file the question, park while it is
9156    /// open, and on an answer, silence or timeout pick the text to post. The
9157    /// decision is saved before it is used, so it applies once.
9158    async fn settle_withheld(
9159        &mut self,
9160        winner: &Candidate,
9161        pr: &PrMessage,
9162        mut g: crate::run::GithubTextAsk,
9163    ) -> Result<Option<PrMessage>> {
9164        use crate::github_text as gt;
9165        let store = ask::Questions::open();
9166        let timeout = self.state.config.graph.answer_timeout;
9167        if !g.resolved {
9168            let mut standing = g
9169                .question
9170                .as_ref()
9171                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9172            if let Some(q) = standing.as_ref()
9173                && q.status.open()
9174                && gt::expired(q, timeout)
9175            {
9176                let why = format!("no answer within {}s of asking", timeout.max(1));
9177                standing = store
9178                    .update(&q.id, |q| {
9179                        q.abandon(&why);
9180                        Ok(())
9181                    })
9182                    .ok()
9183                    .map(|(q, ())| q);
9184            }
9185            let mut ask_again: Option<Option<Vec<String>>> = None;
9186            match standing {
9187                // Recorded but never written (a stop between the two saves),
9188                // or removed: ask again without spending the retry.
9189                None if g.asks <= 1 => {
9190                    g.asks = 0;
9191                    ask_again = Some(None);
9192                }
9193                None => {
9194                    g.resolved = true;
9195                    g.chosen_title = None;
9196                    g.choice = Some(GithubTextChoice::Fallback);
9197                }
9198                Some(q) if q.status.open() => {
9199                    self.park_on_github_text(&g)?;
9200                    return Ok(None);
9201                }
9202                Some(q) => match gt::read_reply(&q) {
9203                    gt::Reply::Fallback => {
9204                        g.resolved = true;
9205                        g.chosen_title = None;
9206                        g.choice = Some(GithubTextChoice::Fallback);
9207                    }
9208                    gt::Reply::Original => {
9209                        g.resolved = true;
9210                        g.chosen_title = None;
9211                        g.choice = Some(GithubTextChoice::Original);
9212                    }
9213                    gt::Reply::Rewrite => {
9214                        g.resolved = true;
9215                        g.chosen_title = None;
9216                        g.choice = Some(if g.rewrite.is_some() {
9217                            GithubTextChoice::Rewrite
9218                        } else {
9219                            GithubTextChoice::Fallback
9220                        });
9221                    }
9222                    gt::Reply::Title(text) => {
9223                        let first = text
9224                            .lines()
9225                            .map(str::trim)
9226                            .find(|l| !l.is_empty())
9227                            .unwrap_or("");
9228                        let decision = if first.is_empty() {
9229                            None
9230                        } else {
9231                            let candidate = PrMessage {
9232                                title: first.to_owned(),
9233                                body: String::new(),
9234                            };
9235                            self.judge_pr_language(&winner.worktree, &candidate).await
9236                        };
9237                        match gt::vet_title(
9238                            &text,
9239                            decision,
9240                            self.state.config.graph.github_english(),
9241                        ) {
9242                            Ok(title) => {
9243                                g.resolved = true;
9244                                g.chosen_title = Some(title);
9245                                g.choice = Some(GithubTextChoice::MyTitle);
9246                            }
9247                            Err(cats) if g.asks < 2 => {
9248                                ask_again = Some(Some(
9249                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9250                                ))
9251                            }
9252                            Err(_) => {
9253                                g.resolved = true;
9254                                g.chosen_title = None;
9255                                g.choice = Some(GithubTextChoice::Fallback);
9256                            }
9257                        }
9258                    }
9259                },
9260            }
9261            if let Some(rejected) = ask_again {
9262                // A retry (after a rejected replacement title) is still about
9263                // the original text: the same ORIGINAL / FALLBACK comparison.
9264                let w = gt::Withheld {
9265                    title: g.title,
9266                    body: g.body,
9267                    categories: g.categories.clone(),
9268                };
9269                // Named by the original text's fingerprint alone, so a resume
9270                // rewrites the same file and the detail stays identical.
9271                let artifact = match crate::run::write_artifact(
9272                    &self.state,
9273                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9274                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9275                ) {
9276                    Ok(p) => Some(p),
9277                    Err(e) => {
9278                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9279                        None
9280                    }
9281                };
9282                let (detail, comparison) = gt::question_content(
9283                    &w,
9284                    &pr.title,
9285                    &pr.body,
9286                    &self.state.id,
9287                    rejected.as_deref(),
9288                    artifact.as_deref(),
9289                    g.rewrite.as_ref(),
9290                );
9291                let mut q = ask::Question::new(
9292                    self.state.id.clone(),
9293                    gt::ASK_NODE.to_owned(),
9294                    gt::ASK_SEAT.to_owned(),
9295                    gt::question_summary(&self.state.config.graph.language, &w),
9296                    detail,
9297                    gt::question_choices(&w, g.rewrite.as_ref()),
9298                );
9299                q.posting_comparison = Some(comparison);
9300                q.answer_timeout = timeout;
9301                // Recorded before it is written: a stop in between leaves a
9302                // record naming a question the resume then files again.
9303                g.question = Some(q.id.clone());
9304                g.asks += 1;
9305                self.state.github_text = Some(g.clone());
9306                self.state.event(
9307                    "github-text",
9308                    format!("asking the owner about the withheld text ({})", q.short()),
9309                );
9310                self.state.save()?;
9311                store
9312                    .put_executable(&mut q)
9313                    .context("file the posting-gate question")?;
9314                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9315                    tracing::warn!(
9316                        "could not notify about posting-gate question {}: {e:#}",
9317                        q.short()
9318                    );
9319                }
9320                self.park_on_github_text(&g)?;
9321                return Ok(None);
9322            }
9323        }
9324        let resolved = g.chosen_title.clone();
9325        let original = g.choice == Some(GithubTextChoice::Original);
9326        // Field names only: the event log never records the text.
9327        self.state.event(
9328            "github-text",
9329            if original {
9330                format!(
9331                    "posting the original text for: {}",
9332                    match (g.title, g.body) {
9333                        (true, true) => "title, description",
9334                        (true, false) => "title",
9335                        _ => "description",
9336                    }
9337                )
9338            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9339                "posting the saved rewrite title and description".to_owned()
9340            } else if resolved.is_some() {
9341                "posting the owner's replacement title".to_owned()
9342            } else {
9343                "posting the neutral text for the withheld fields".to_owned()
9344            },
9345        );
9346        self.state.github_text = Some(g.clone());
9347        self.state.save()?;
9348        if g.choice == Some(GithubTextChoice::Rewrite)
9349            && let Some(r) = g.rewrite.as_ref()
9350        {
9351            return Ok(Some(PrMessage {
9352                title: r.title.clone(),
9353                body: r.body.clone(),
9354            }));
9355        }
9356        let w = gt::Withheld {
9357            title: g.title,
9358            body: g.body,
9359            categories: g.categories.clone(),
9360        };
9361        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9362        if let Some(t) = resolved {
9363            title = t;
9364        }
9365        // Every withheld field is now neutral or vetted English.
9366        let english = gt::LanguageDecision {
9367            title_english: true,
9368            body_english: true,
9369            body_complete: true,
9370        };
9371        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9372        if original {
9373            // The owner's explicit choice overrides the gate for the fields it
9374            // was made for; the others keep what the gate produced.
9375            if g.title {
9376                title = pr.title.clone();
9377            }
9378            if g.body {
9379                body = pr.body.clone();
9380            }
9381        }
9382        Ok(Some(PrMessage { title, body }))
9383    }
9384
9385    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9386        self.state.github_text = Some(g.clone());
9387        self.state.end_pass_parked(ParkCause::Question);
9388        self.state.parked = true;
9389        self.state.event(
9390            "github-text",
9391            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9392        );
9393        self.state.save()
9394    }
9395
9396    /// Ask `[roles] language_judge` about this text and record which source
9397    /// decided; the text itself is never recorded.
9398    async fn judge_pr_language(
9399        &mut self,
9400        cwd: &Path,
9401        pr: &PrMessage,
9402    ) -> Option<crate::github_text::LanguageDecision> {
9403        self.state.config.roles.language_judge.as_ref()?;
9404        if !self.state.config.graph.language_gate() {
9405            return None;
9406        }
9407        let decision =
9408            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9409        self.state.event(
9410            "github-text",
9411            if decision.is_some() {
9412                "language decided by the language judge"
9413            } else {
9414                "language judge unavailable; using built-in heuristics"
9415            },
9416        );
9417        decision
9418    }
9419
9420    async fn merge(&mut self) -> Result<()> {
9421        if self.resource_halt()? || self.hold_base_sync().await? {
9422            return Ok(());
9423        }
9424        // Same reasoning as `gate`: ask the review and gate records directly
9425        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9426        // skip can rewrite on reentry to something that no longer says
9427        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9428        // so a hand-off (open findings, green verification) reaches merge
9429        // exactly like a genuinely clean round does.
9430        //
9431        // A run resumed mid-`land` never reaches here at all: `execute`
9432        // recognises `RunStatus::Landing` before it even calls `prep`, and
9433        // routes straight to `run_land` instead. That has to happen a level
9434        // up from this function, not with a check in here, because
9435        // `review_loop`'s own status recomputation (see its doc) runs
9436        // *before* `merge` on every reentry and would otherwise overwrite
9437        // the `Landing` marker with `Gating` before this node ever saw it.
9438        if self
9439            .state
9440            .base_sync
9441            .as_ref()
9442            .is_some_and(|s| s.conflict.is_some())
9443            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9444                != Some(RunStatus::Gating)
9445            // `gate_ran == false` is not "passed" - `gate` leaves it false
9446            // both before it has ever run and when its last attempt was
9447            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9448            // permission to merge on nothing but the review record. Only a
9449            // gate that actually ran may proceed; an intentional local
9450            // review can also finish with verification explicitly not configured.
9451            // `RunState::gate_status` computes that distinction.
9452            || !self.state.gate_allows_progress()
9453        {
9454            return Ok(());
9455        }
9456        // This node's own record, not `status`: `status == Ready` is not
9457        // unique to the harmless `MergeMode::None` path this line was
9458        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9459        // run's PR was closed without merging — and on that run `mode` is
9460        // still `Pr`, so a reentry that fell through here would push and
9461        // open a second pull request. `self.state.merge` is set exactly once
9462        // this node (or `land`) has already produced a verdict, under every
9463        // mode, which is what "already done" actually means here.
9464        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9465            return Ok(());
9466        }
9467        let Some(winner) = self.state.winner().cloned() else {
9468            return Ok(());
9469        };
9470        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9471        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9472            self.state.status = RunStatus::Blocked;
9473            self.state.event(
9474                "gate",
9475                "winner changed after gate; fresh verification required",
9476            );
9477            self.state.save()?;
9478            return Ok(());
9479        }
9480        let wait = self.repository_wait();
9481        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9482            return Ok(());
9483        }
9484        let repo = self.state.repo.clone();
9485        let base = self.state.base_branch.clone();
9486        let mode = self.state.config.merge.mode;
9487        let style = self.state.config.merge.style;
9488        let facts = if is_review_run(&self.state) {
9489            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9490            let start = review_base(
9491                &repo,
9492                &self.state.config.merge.remote,
9493                &base,
9494                &self.state.base_commit,
9495                &winner.branch,
9496            )
9497            .await;
9498            branch_facts(&repo, &start, &winner.branch).await
9499        } else {
9500            None
9501        };
9502        // `None` when the base could not be freshly read: then an adopted
9503        // pull request's title is left alone.
9504        let leaked = if is_review_run(&self.state) {
9505            leaked_subjects(&self.state, &winner.branch).await
9506        } else {
9507            Some(Vec::new())
9508        };
9509        // `None`: parked on the owner's word about a withheld title or body.
9510        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9511        // the resume arrives here again.
9512        let Some(pr) = self
9513            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9514            .await?
9515        else {
9516            return Ok(());
9517        };
9518        let message = pr.commit_message();
9519
9520        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9521        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9522            return Ok(());
9523        }
9524        let outcome = match mode {
9525            MergeMode::None => MergeOutcome {
9526                mode,
9527                ok: true,
9528                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9529                empty: false,
9530            },
9531            MergeMode::Pr | MergeMode::Local
9532                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9533            {
9534                MergeOutcome {
9535                    mode,
9536                    ok: false,
9537                    detail: empty_candidate_detail(&self.state, &base),
9538                    empty: true,
9539                }
9540            }
9541            MergeMode::Local => {
9542                let on = git::current_branch(&repo).await?;
9543                if on.as_deref() != Some(base.as_str()) {
9544                    MergeOutcome {
9545                        mode,
9546                        ok: false,
9547                        detail: format!(
9548                            "{} has {} checked out, not the base branch {base}",
9549                            repo.display(),
9550                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9551                        ),
9552                        empty: false,
9553                    }
9554                } else if !git::is_clean(&repo).await? {
9555                    MergeOutcome {
9556                        mode,
9557                        ok: false,
9558                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9559                        empty: false,
9560                    }
9561                } else {
9562                    let out = match style {
9563                        MergeStyle::Merge => {
9564                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9565                        }
9566                        MergeStyle::Squash => {
9567                            git::merge_squash(&repo, &winner.branch, &message).await?
9568                        }
9569                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9570                    };
9571                    MergeOutcome {
9572                        mode,
9573                        ok: out.ok(),
9574                        detail: if out.ok() { out.stdout } else { out.stderr },
9575                        empty: false,
9576                    }
9577                }
9578            }
9579            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9580                self.state.pr_publication_pending = true;
9581                self.state.save()?;
9582                let out =
9583                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9584                self.state.pr_publication_pending = out.is_err();
9585                MergeOutcome {
9586                    mode,
9587                    ok: out.is_ok(),
9588                    detail: match out {
9589                        Ok(url) => url,
9590                        Err(e) => format!("{e:#}"),
9591                    },
9592                    empty: false,
9593                }
9594            }
9595            MergeMode::Pr => {
9596                let remote = self.state.config.merge.remote.clone();
9597                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9598                let pushed = if self.state.foreign_checkout.is_some() {
9599                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9600                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9601                } else {
9602                    git::push(&winner.worktree, &remote, &winner.branch).await?
9603                };
9604                if !pushed.ok() {
9605                    MergeOutcome {
9606                        mode,
9607                        ok: false,
9608                        detail: pushed.stderr,
9609                        empty: false,
9610                    }
9611                } else {
9612                    if self.state.foreign_checkout.is_some() {
9613                        self.state.pr_pushed_head = Some(delivery_head.clone());
9614                    }
9615                    // A retry or resume of a run whose branch already has an
9616                    // open pull request adopts it rather than failing on a
9617                    // duplicate. Only this winner branch into this base:
9618                    // `branch_for` derives the name from the run id, so a
9619                    // different run's pull request never matches.
9620                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9621                    let out = match pr_merge_plan(found) {
9622                        PrPlan::Create => {
9623                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9624                                .await
9625                        }
9626                        PrPlan::Adopt { url, title } => {
9627                            self.state
9628                                .event("merge", format!("Pr: adopted open pull request {url}"));
9629                            if self.state.github_text.as_ref().is_some_and(|g| {
9630                                g.resolved
9631                                    && g.choice == Some(GithubTextChoice::Rewrite)
9632                                    && g.rewrite.is_some()
9633                            }) {
9634                                // The owner chose the entire pair, even if the
9635                                // existing title already matches or was edited.
9636                                land::set_pr_text_verbatim(
9637                                    &winner.worktree,
9638                                    &url,
9639                                    &pr.title,
9640                                    &pr.body,
9641                                )
9642                                .await
9643                                .map(|()| url)
9644                            } else {
9645                                if title != pr.title
9646                                    && (!is_review_run(&self.state)
9647                                        || leaked
9648                                            .as_deref()
9649                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9650                                    && let Err(e) =
9651                                        if self.state.github_text.as_ref().is_some_and(|g| {
9652                                            g.resolved
9653                                                && g.title
9654                                                && g.choice == Some(GithubTextChoice::Original)
9655                                        }) {
9656                                            // The owner chose to post this title unmodified.
9657                                            land::set_pr_title_verbatim(
9658                                                &winner.worktree,
9659                                                &url,
9660                                                &pr.title,
9661                                            )
9662                                            .await
9663                                        } else {
9664                                            land::set_pr_title(
9665                                                &mut self.state,
9666                                                &winner.worktree,
9667                                                &url,
9668                                                &pr.title,
9669                                            )
9670                                            .await
9671                                        }
9672                                {
9673                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9674                                    self.state
9675                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9676                                }
9677                                Ok(url)
9678                            }
9679                        }
9680                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9681                    };
9682                    match out {
9683                        Ok(url) => MergeOutcome {
9684                            mode,
9685                            ok: true,
9686                            detail: url,
9687                            empty: false,
9688                        },
9689                        Err(e) => MergeOutcome {
9690                            mode,
9691                            ok: false,
9692                            detail: e.to_string(),
9693                            empty: false,
9694                        },
9695                    }
9696                }
9697            }
9698        };
9699
9700        self.state.status = match (mode, outcome.ok) {
9701            (MergeMode::None, _) => RunStatus::Ready,
9702            (_, true) if outcome.empty => RunStatus::Ready,
9703            (MergeMode::Pr, true) => RunStatus::Ready,
9704            (_, true) => RunStatus::Merged,
9705            (_, false) => RunStatus::Blocked,
9706        };
9707        self.state.event(
9708            "merge",
9709            format!(
9710                "{:?}: {}",
9711                mode,
9712                outcome.detail.lines().next().unwrap_or("")
9713            ),
9714        );
9715        self.state.merge = Some(outcome);
9716        self.state.save()?;
9717
9718        // The PR is open and the run would historically stop here, leaving the
9719        // operator to watch checks, feed review comments back to a fixer, and
9720        // merge. That was done by hand six times in one session before this
9721        // existed. Opt-in, because merging is the one irreversible thing magi
9722        // can do to a repository.
9723        if self.state.config.graph.land
9724            && mode == MergeMode::Pr
9725            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9726        {
9727            self.run_land().await?;
9728        }
9729        // `run_land` may have left `status` at `Landing` - still waiting on
9730        // CI or the owner's approval, not actually settled - so this has to
9731        // read whatever `status` ended up as here, not the `Merged` this
9732        // function set a few lines up.
9733        self.settle_questions();
9734        Ok(())
9735    }
9736
9737    /// Enter `land`.
9738    ///
9739    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9740    /// a resumed run's re-entry. `land::land` itself is what serialises the
9741    /// two git-mutating moments inside the loop — the rebase push and
9742    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9743    /// to hold a lock across the whole call, and doing so would serialise
9744    /// this run's CI wait against a *different* run's land-approval resume
9745    /// in the same repository, which is exactly the "must not wait on
9746    /// another task" property the daemon's slot-freeing exists to give.
9747    async fn run_land(&mut self) -> Result<()> {
9748        crate::pr_head::ensure_owner(&self.state)?;
9749        let url = self
9750            .state
9751            .pr
9752            .as_ref()
9753            .map(|pr| pr.url.clone())
9754            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9755            .unwrap_or_default();
9756        let url = url.lines().next().unwrap_or("").trim().to_owned();
9757        if !url.starts_with("http") {
9758            return Ok(());
9759        }
9760        // A land failure is not a lost run: the work is on a branch and the
9761        // pull request is open, which is exactly where a human takes over.
9762        loop {
9763            match land::land(&mut self.state, &url).await {
9764                Ok(_)
9765                    if self
9766                        .state
9767                        .merge_repairs
9768                        .last()
9769                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9770                {
9771                    if self.repair_merge_review().await? {
9772                        continue;
9773                    }
9774                }
9775                Ok(_) if self.state.resource_block.is_some() => {
9776                    return self.stop_for_resource();
9777                }
9778                Ok(pr) if self.state.parked => {
9779                    // `land` already saved the parked marker; nothing here
9780                    // overrides `status` back to a terminal value while an
9781                    // approval is still outstanding.
9782                    let _ = pr;
9783                }
9784                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9785                    self.state.status = RunStatus::Superseded;
9786                    self.state.save()?;
9787                }
9788                Ok(pr) => {
9789                    self.state.status = match pr.state {
9790                        land::PrLifecycle::Merged => RunStatus::Merged,
9791                        _ => RunStatus::Blocked,
9792                    };
9793                    // Downstream of a confirmed merge only - see
9794                    // `bump::should_release_bump`'s own doc for why this one
9795                    // check covers all three of `land`'s success paths.
9796                    // Best-effort: the run already landed, so a failure here
9797                    // (the decision call, `gh`, `cargo`) is recorded and never
9798                    // turns a landed run into a failed one.
9799                    if bump::should_release_bump(self.state.status)
9800                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9801                    {
9802                        // The event is the run's own record. Not-eligible cases
9803                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9804                        // `Err` is a bump that was tried and failed:
9805                        // `after_merge` itself raises the operator notice for
9806                        // that, whether or not a release PR exists yet.
9807                        self.state
9808                            .event("bump", format!("release bump skipped: {e:#}"));
9809                    }
9810                    // Independent of the bump, and best-effort in the same way:
9811                    // findings the merge left open become follow-up tasks.
9812                    if self.state.status == RunStatus::Merged {
9813                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9814                    }
9815                    self.state.save()?;
9816                }
9817                Err(e) => {
9818                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9819                        return Err(e);
9820                    }
9821                    self.state.status = RunStatus::Blocked;
9822                    self.state.event("land", format!("gave up: {e}"));
9823                    self.state.save()?;
9824                }
9825            }
9826            break;
9827        }
9828        Ok(())
9829    }
9830
9831    /// One repair cycle stays under the existing driver/task claim. An
9832    /// interrupted fixer is rolled back and held rather than run twice.
9833    async fn repair_merge_review(&mut self) -> Result<bool> {
9834        use crate::run::MergeRepairPhase as Phase;
9835        crate::pr_head::ensure_owner(&self.state)?;
9836        let i = self.state.merge_repairs.len() - 1;
9837        let request = self.state.merge_repairs[i].clone();
9838        if request.phase == Phase::Fixing {
9839            let restored = match self.state.winner().cloned() {
9840                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9841                None => Err(anyhow::anyhow!("repair has no winning branch")),
9842            };
9843            let why = match restored {
9844                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9845                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9846                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9847            };
9848            self.hold_merge_repair(i, why)?;
9849            return Ok(false);
9850        }
9851        // A completed repair gate is a publication checkpoint. On reentry,
9852        // reject external changes before the ordinary gate path can replace
9853        // that evidence by verifying a new head. Requested repairs still
9854        // start their own review generation below.
9855        if request.phase == Phase::Reviewing && self.state.gate_ran {
9856            let winner = self
9857                .state
9858                .winner()
9859                .context("repair has no winning branch")?;
9860            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9861            if self.state.gate_head.as_deref() != Some(head.as_str())
9862                || !git::is_clean(&winner.worktree).await?
9863            {
9864                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9865                return Ok(false);
9866            }
9867        }
9868        let failure = if i >= self.state.config.graph.land_rounds
9869            || self.state.config.graph.review_rounds == 0
9870        {
9871            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9872        } else {
9873            None
9874        };
9875        if let Some(why) = failure {
9876            self.hold_merge_repair(i, why)?;
9877            return Ok(false);
9878        }
9879        if request.phase == Phase::Requested {
9880            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9881                Ok(pr) => pr,
9882                Err(e) => {
9883                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9884                    return Ok(false);
9885                }
9886            };
9887            self.state.merge_repairs[i].phase = Phase::Fixing;
9888            self.state.land_approval = None;
9889            self.state.land_armed_head = None;
9890            self.state.save()?;
9891            // Serialize the exact records, including finding ids/details,
9892            // reconsideration reasons, final votes and verification output.
9893            let evidence = serde_json::to_string_pretty(&request)?;
9894            let reason = format!(
9895                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9896            );
9897            let budget = self.state.config.graph.land_rounds;
9898            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9899            match result {
9900                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9901                Ok(land::Fixed::Deferred) => {
9902                    // An unavailable fixer did not spend this repair request.
9903                    // Restore partial fallback work before retrying it, as for
9904                    // an interrupted repair, without closing the request.
9905                    let worktree = self
9906                        .state
9907                        .winner()
9908                        .context("repair has no winning branch")?
9909                        .worktree
9910                        .clone();
9911                    self.graft_restore(&worktree, &request.head).await?;
9912                    self.state.merge_repairs[i].phase = Phase::Requested;
9913                    self.stop_for_resource()?;
9914                    return Ok(false);
9915                }
9916                Ok(land::Fixed::Failed(why)) => {
9917                    self.hold_merge_repair(
9918                        i,
9919                        format!(
9920                            "repair failed: {why}; inspect the branch and run an explicit review"
9921                        ),
9922                    )?;
9923                    return Ok(false);
9924                }
9925                Err(e) => {
9926                    self.hold_merge_repair(
9927                        i,
9928                        format!(
9929                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9930                        ),
9931                    )?;
9932                    return Ok(false);
9933                }
9934            }
9935            // Even no change gets a real new reviewer wave. The previous
9936            // evidence remains in the request; a former clean/concluded round
9937            // cannot short-circuit the new review or gate.
9938            self.begin_merge_repair_review(i)?;
9939        }
9940        self.review_loop().await?;
9941        self.gate().await?;
9942        if self.resource_halt()? || self.state.parked {
9943            return Ok(false);
9944        }
9945        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9946            != Some(RunStatus::Gating)
9947            || !self.state.gate_allows_progress()
9948        {
9949            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9950            return Ok(false);
9951        }
9952        // Review and gate may have added commits after the initial repair.
9953        // Publish exactly their final head to the existing PR, with a lease.
9954        let winner = self
9955            .state
9956            .winner()
9957            .cloned()
9958            .context("repair has no winning branch")?;
9959        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9960        if self.state.gate_head.as_deref() != Some(after.as_str())
9961            || !git::is_clean(&winner.worktree).await?
9962        {
9963            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9964            return Ok(false);
9965        }
9966        let expected = self
9967            .state
9968            .pr_pushed_head
9969            .as_deref()
9970            .unwrap_or(&request.head)
9971            .to_owned();
9972        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9973        // Push the checked commit itself: a local ref moving after the check
9974        // must not change which commit the forge receives.
9975        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9976            return Ok(false);
9977        }
9978        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9979        let refspec = format!("{after}:refs/heads/{public}");
9980        let push = git::git_raw(
9981            &winner.worktree,
9982            &["push", &lease, &self.state.config.merge.remote, &refspec],
9983        )
9984        .await;
9985        match push {
9986            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9987            Ok(out) => {
9988                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9989                return Ok(false);
9990            }
9991            Err(e) => {
9992                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9993                return Ok(false);
9994            }
9995        }
9996        self.state.merge_repairs[i].phase = Phase::Complete;
9997        self.state.status = RunStatus::Landing;
9998        self.state.event(
9999            "repair",
10000            "new review and gate completed; fresh owner merge decision required",
10001        );
10002        self.state.save()?;
10003        Ok(true)
10004    }
10005
10006    fn merge_repair_artifacts(&self) -> PathBuf {
10007        let artifacts = agent::artifacts_dir(&self.state.dir());
10008        match self
10009            .state
10010            .merge_repairs
10011            .last()
10012            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10013        {
10014            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10015            None => artifacts,
10016        }
10017    }
10018
10019    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10020        if let Some(e2e) = self.state.head_e2e.take() {
10021            self.state
10022                .verification_history
10023                .push(crate::run::VerificationEvidence {
10024                    superseded_by: None,
10025                    archived_at: Some(Timestamp::now()),
10026                    fixes_spent: Some(self.state.gate_fixes.len()),
10027                    gate_head: None,
10028                    gate_ran: false,
10029                    gate: Vec::new(),
10030                    e2e: Some(e2e),
10031                });
10032        }
10033        self.state.reviews.clear();
10034        self.state.gate.clear();
10035        self.state.gate_ran = false;
10036        self.state.gate_head = None;
10037        self.state.gate_fixes.clear();
10038        self.state.gate_repair_e2e = None;
10039        self.state.pre_gate.clear();
10040        self.state.pre_gate_commit = None;
10041        self.state.contested_handoff = None;
10042        self.state.land_reruns = None;
10043        self.state.land_approval = None;
10044        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10045        self.state.status = RunStatus::Reviewing;
10046        self.state.save()
10047    }
10048
10049    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10050        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10051        self.state.merge_repairs[index].failure = Some(why.clone());
10052        self.state.status = RunStatus::Blocked;
10053        self.state.event("repair", why);
10054        self.state.save()
10055    }
10056
10057    // -------------------------------------------------------------- helpers
10058
10059    /// Fetch or create a seat, keeping its conversation across nodes.
10060    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10061        if let Some(existing) = self.state.seats.get(key)
10062            && existing.agent == agent
10063        {
10064            return existing.clone();
10065        }
10066        // A seat that changes agent mints its session id from the agent too,
10067        // like a handover: the old agent's uuid is already taken by the CLI.
10068        let fresh = if self.state.seats.contains_key(key) {
10069            handover_seat(key, agent, self.state.next_seat_seed())
10070        } else {
10071            SeatState::new(key, agent, self.state.seed)
10072        };
10073        self.state.seats.insert(key.to_owned(), fresh.clone());
10074        fresh
10075    }
10076
10077    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10078    /// seat to another roster agent, in which case that agent. Nodes that
10079    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10080    /// reconsideration) must keep talking to whoever answered it, not slip
10081    /// back to the agent that failed it.
10082    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10083        match self.state.seats.get(key) {
10084            Some(s) if s.agent != spec.id => {
10085                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10086            }
10087            _ => spec,
10088        }
10089    }
10090
10091    /// A candidate rendered for judging, with the leak policy applied.
10092    fn view(&self, c: &Candidate) -> CandidateView {
10093        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10094            .unwrap_or_default();
10095        let (patch, _) = blind::sanitize_patch(
10096            &format!("candidate {} patch", c.label),
10097            &raw,
10098            &self.state.config.blind,
10099        );
10100        CandidateView {
10101            label: c.label,
10102            branch: c.branch.clone(),
10103            summary: c.summary.clone(),
10104            stat: c.stat.clone(),
10105            patch,
10106        }
10107    }
10108
10109    /// The full candidate set as prompt text, for seats with no live session.
10110    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10111        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10112        prompt::judge(
10113            "(see above)",
10114            &views,
10115            self.roles.judges.len(),
10116            base_short,
10117            "en",
10118        )
10119    }
10120
10121    /// The final-vote prompt with everything a seat that has no session of its
10122    /// own needs: the candidates, the seat's own ranking and reasons, and the
10123    /// anonymised deliberation it took part in (only when there was one, so a
10124    /// handed-over seat never sees more than the seat it replaces did). The
10125    /// `Final vote` heading stays first.
10126    fn vote_prompt_full(
10127        &self,
10128        j: usize,
10129        labels: &[char],
10130        language: &str,
10131        candidates: &[Candidate],
10132        base_short: &str,
10133    ) -> String {
10134        let mut text = format!(
10135            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10136            prompt::final_vote(labels, language),
10137            self.state.instruction,
10138            self.candidate_block(candidates, base_short)
10139        );
10140        if let Some(own) = self
10141            .state
10142            .judgements
10143            .get(j)
10144            .filter(|r| !r.ranking.is_empty())
10145        {
10146            let reasons = own
10147                .reasons
10148                .iter()
10149                .map(|(k, v)| format!("- {k}: {v}"))
10150                .collect::<Vec<_>>()
10151                .join("\n");
10152            text.push_str(&format!(
10153                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10154                own.ranking.iter().collect::<String>(),
10155                if reasons.is_empty() {
10156                    ""
10157                } else {
10158                    ", because:\n"
10159                }
10160            ));
10161        }
10162        if !self.state.deliberation.is_empty() {
10163            text.push_str("\n# What was argued before this vote\n");
10164            for t in self.transcript(&[], j) {
10165                text.push_str(&format!(
10166                    "\n## {}{}\n\n{}\n",
10167                    t.who,
10168                    if t.is_self { " (you)" } else { "" },
10169                    t.body.trim()
10170                ));
10171            }
10172        }
10173        text
10174    }
10175
10176    /// Anonymised transcript for judge `self_idx`.
10177    ///
10178    /// The initial rankings are always the opening statements. Seeding them
10179    /// only when no turn had been taken yet meant every judge after the first
10180    /// argued against a single voice instead of against the actual split — the
10181    /// disagreement is the information, so it is always on the table.
10182    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10183        let mut turns = Vec::new();
10184        for j in &self.state.judgements {
10185            if j.ranking.is_empty() {
10186                continue;
10187            }
10188            let reasons = j
10189                .reasons
10190                .iter()
10191                .map(|(k, v)| format!("- {k}: {v}"))
10192                .collect::<Vec<_>>()
10193                .join("\n");
10194            turns.push(Turn {
10195                who: format!("Judge {} (opening ranking)", j.judge),
10196                is_self: j.judge == self_idx + 1,
10197                body: format!(
10198                    "Ranked {}{}{reasons}",
10199                    j.ranking.iter().collect::<String>(),
10200                    if reasons.is_empty() {
10201                        ""
10202                    } else {
10203                        ", because:\n"
10204                    }
10205                ),
10206            });
10207        }
10208        for t in self
10209            .state
10210            .deliberation
10211            .iter()
10212            .flat_map(|r| r.turns.iter())
10213            .chain(current)
10214        {
10215            turns.push(Turn {
10216                who: format!("Judge {}", t.judge),
10217                is_self: t.judge == self_idx + 1,
10218                body: t.body.clone(),
10219            });
10220        }
10221        turns
10222    }
10223}
10224
10225/// Does this seat still hold the context a follow-up prompt would rely on?
10226fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10227    agent::has_session(spec.kind, seat, sessions)
10228}
10229
10230/// The next entry in `roster` after `start`, never wrapping back to the
10231/// front, whose id is not in `tried` yet.
10232///
10233/// Starts one past `start` rather than at the front of `roster`: `start` is
10234/// the seat's own original position, and a seat whose candidate slot already
10235/// sits on the roster's second entry must fall through to the third next, not
10236/// restart at the first — which is very likely a different candidate's own
10237/// agent already. Never wraps back past `start`, for the same reason: an
10238/// entry earlier in the roster than the seat's own position is almost
10239/// certainly some *other* candidate slot's own agent, and once the tail of
10240/// the roster is exhausted there are no more untried agents for *this* seat
10241/// to fall through to — the caller's fallback chain ends there, exactly as
10242/// "no further untried agents remain in the list for that seat" asks for.
10243///
10244/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10245/// the same id twice (an operator's `roles.implementers` typo, or a
10246/// `[[agents]]` list reused across roles) must not let
10247/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10248/// over `roster` either finds an untried id or runs out, so this always
10249/// terminates regardless of duplicates.
10250fn next_untried_in_roster<'a>(
10251    roster: &'a [AgentSpec],
10252    start: usize,
10253    tried: &BTreeSet<String>,
10254) -> Option<&'a AgentSpec> {
10255    roster
10256        .get(start + 1..)?
10257        .iter()
10258        .find(|s| !tried.contains(&s.id))
10259}
10260
10261/// The successor for a seat, shared by all four seat kinds (implement, judge,
10262/// review, advise). `others` holds the ids that *currently* occupy the other
10263/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10264/// sees them), so roster entries beyond the seat count act as spares: a failed
10265/// seat goes to an agent no other seat holds whenever the roster permits.
10266///
10267/// `carried` is `Some` only for the review loop's carried failure history.
10268/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10269/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10270/// whole roster: untried and not another seat's occupant; (3) the plain walk
10271/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10272/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10273/// a better outcome than an empty seat, and it keeps the answer to "is there
10274/// a successor at all" exactly what it was before occupants were considered,
10275/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10276/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10277/// carried failure is a cheaper bet than doubling an agent on the panel, and
10278/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10279/// the same round are handled one at a time, so a seat later in the batch
10280/// sees an earlier one's new occupant but not yet its own freed agent.
10281fn pick_successor<'a>(
10282    roster: &'a [AgentSpec],
10283    start: usize,
10284    tried: &BTreeSet<String>,
10285    carried: Option<&BTreeSet<String>>,
10286    others: &BTreeSet<String>,
10287) -> Option<&'a AgentSpec> {
10288    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10289    roster
10290        .get(start + 1..)
10291        .and_then(|tail| {
10292            tail.iter()
10293                .filter(free)
10294                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10295        })
10296        .or_else(|| {
10297            carried
10298                .is_some()
10299                .then(|| roster.iter().find(free))
10300                .flatten()
10301        })
10302        .or_else(|| match carried {
10303            Some(c) => next_for_seat(roster, start, tried, c),
10304            None => next_untried_in_roster(roster, start, tried),
10305        })
10306}
10307
10308/// The next agent for a seat that carries its failure history across rounds
10309/// (the review loop). `round_tried` is this round's own bound and starts
10310/// empty every round; `carried_failed` only decides priority.
10311///
10312/// First: an id walking forward from `start`, never wrapping, that is neither
10313/// tried this round nor failed in an earlier one. Only when that is exhausted
10314/// does it rescue: the first roster id (in roster order, so this one *does*
10315/// look before `start`) not yet tried this round, which is by then a carried
10316/// failure. Each id is rescued at most once per round, so it cannot loop.
10317fn next_for_seat<'a>(
10318    roster: &'a [AgentSpec],
10319    start: usize,
10320    round_tried: &BTreeSet<String>,
10321    carried_failed: &BTreeSet<String>,
10322) -> Option<&'a AgentSpec> {
10323    roster
10324        .get(start + 1..)?
10325        .iter()
10326        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10327        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10328}
10329
10330/// Where a reviewer seat starts a round: the agent that last answered it when
10331/// it is still on the roster and not marked failed, else the spec's own agent
10332/// unless it failed, else the next roster agent that has not failed, else the
10333/// spec's own agent again (the whole roster failed). Ids no longer on the
10334/// roster are ignored. An empty roster has no handover, so the spec stands.
10335fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10336    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10337        return spec;
10338    };
10339    let ok = |id: &str| !h.failed.contains(id);
10340    if let Some(last) = h.last_ok.as_deref()
10341        && ok(last)
10342        && let Some(s) = roster.iter().find(|s| s.id == last)
10343    {
10344        return s.clone();
10345    }
10346    if ok(&spec.id) {
10347        return spec;
10348    }
10349    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10350    roster
10351        .get(from + 1..)
10352        .into_iter()
10353        .flatten()
10354        .chain(roster.iter())
10355        .find(|s| ok(&s.id))
10356        .cloned()
10357        .unwrap_or(spec)
10358}
10359
10360/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10361/// seed so a CLI that mints its session id up front (`--session-id`) never
10362/// reuses the uuid the previous agent already opened under the same seat key.
10363pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10364    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10365}
10366
10367/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10368/// so the classifier below and the code that builds the message cannot drift.
10369const TIMED_OUT: &str = "timed out";
10370
10371impl FailClass {
10372    /// `None` for an answer; otherwise how the turn failed.
10373    fn of(out: &AgentOutcome) -> Option<Self> {
10374        match out {
10375            AgentOutcome::Ok(_) => None,
10376            AgentOutcome::Quota(_) => Some(Self::Quota),
10377            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10378            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10379            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10380            // Not a failure of the agent. Callers check for it first; `None`
10381            // here means "no class to hand over on".
10382            AgentOutcome::ResourceBlocked(_) => None,
10383        }
10384    }
10385
10386    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10387    fn stem_word(&self) -> &'static str {
10388        match self {
10389            Self::Quota => "quota",
10390            _ => "handover",
10391        }
10392    }
10393}
10394
10395/// The message's first line with its variable parts removed — digit runs and
10396/// path-like tokens — so "exit code 2" and "exit code 7" read
10397/// as one kind of failure.
10398fn failure_signature(msg: &str) -> String {
10399    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10400    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10401    // the historical machine key while the process diagnosis uses readable prose.
10402    if let Some(code) = line
10403        .strip_prefix("process exited with no usable output (exit code ")
10404        .and_then(|s| s.strip_suffix(')'))
10405        .and_then(|s| s.parse::<i32>().ok())
10406    {
10407        line = format!("exited with some({code}) and no usable output");
10408    } else if matches!(
10409        line.as_str(),
10410        "process exited with no usable output (no exit code reported)"
10411            | "process exited with no usable output (exit code unavailable)"
10412    ) {
10413        line = "exited with none and no usable output".to_owned();
10414    }
10415    let mut out = Vec::new();
10416    for word in line.split_whitespace() {
10417        if word.contains('/') || word.contains('\\') {
10418            out.push("<path>".to_owned());
10419            continue;
10420        }
10421        let mut w = String::new();
10422        let mut in_digits = false;
10423        for c in word.chars() {
10424            if c.is_ascii_digit() {
10425                if !in_digits {
10426                    w.push('#');
10427                }
10428                in_digits = true;
10429            } else {
10430                in_digits = false;
10431                w.push(c);
10432            }
10433        }
10434        out.push(w);
10435    }
10436    out.join(" ").chars().take(120).collect()
10437}
10438
10439/// Whether a seat that just failed with `cur` may go to the next roster agent.
10440/// A quota or a timeout always may. Any other failure may not when the agent
10441/// before it failed the same way: an error the prompt causes would otherwise
10442/// walk the whole roster. `prev` is the class of the immediately preceding
10443/// agent's failure, so a quota or timeout in between breaks the run of
10444/// identical failures by itself.
10445fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10446    match cur {
10447        FailClass::Quota | FailClass::Timeout => true,
10448        FailClass::Other(_) => prev != Some(cur),
10449    }
10450}
10451
10452/// A short human reason for a failed outcome, for the handover record.
10453fn fail_reason(out: &AgentOutcome) -> String {
10454    match out {
10455        AgentOutcome::Ok(_) => String::new(),
10456        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10457        AgentOutcome::Dropped(o) => format!(
10458            "the CLI dropped the stream ({})",
10459            o.dropped
10460                .as_ref()
10461                .map(|d| d.why.as_str())
10462                .unwrap_or("it ended without delivering its answer")
10463        ),
10464        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10465            e.lines().next().unwrap_or("").chars().take(160).collect()
10466        }
10467    }
10468}
10469
10470/// Note one handover in the run: the structured record and, in the timeline,
10471/// the sentence a person reads. A quota keeps the wording it always had.
10472pub(crate) fn record_handover(
10473    state: &mut RunState,
10474    node: &str,
10475    seat: &str,
10476    from: &str,
10477    to: &str,
10478    class: &FailClass,
10479    reason: &str,
10480) {
10481    let message = if *class == FailClass::Quota {
10482        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10483    } else {
10484        format!("{seat}: handed over {from} -> {to} ({reason})")
10485    };
10486    state.event(node, message);
10487    state.handovers.push(Handover {
10488        at: Timestamp::now(),
10489        node: node.to_owned(),
10490        seat: seat.to_owned(),
10491        from: from.to_owned(),
10492        to: to.to_owned(),
10493        reason: reason.to_owned(),
10494    });
10495}
10496
10497/// Did this reply report running a command whose own CLI never confirmed an
10498/// exit status?
10499///
10500/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10501/// command *finished* (see that type's own doc), so this can only be `true`
10502/// for a command whose completion event carried no readable exit code — not
10503/// for one that simply is not mentioned at all. That is the one signal this
10504/// crate can read, from the same record `state.jobs` renders, about a reply
10505/// standing next to work its own CLI cannot vouch for finishing; it is
10506/// deliberately not a check on the exit code's *value* (a fixer legitimately
10507/// runs a command that fails mid-iteration before it succeeds) and not a
10508/// guess at a command still running in the background (which emits no event
10509/// at all, and so leaves no evidence here to find).
10510fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10511    commands.iter().any(|c| c.exit_code.is_none())
10512}
10513
10514/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10515/// trusted as a verified no-op — the adoption guard's own text-level half.
10516///
10517/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10518/// timed out): a marker only earns the benefit of the doubt from a turn the
10519/// CLI itself vouches for finishing properly, the same house style
10520/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10521/// *fix* report to for `commands`. A candidate that timed out, exited
10522/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10523/// is, whatever prose it wrote — this returns `None` before it ever looks at
10524/// `text`. The remaining guards (the tree really is empty, the evidence is
10525/// non-empty) are the caller's: this only reads what the reply *claimed*.
10526fn verified_noop_claim(
10527    usable: bool,
10528    commands: &[agent::CommandEvidence],
10529    text: &str,
10530) -> Option<String> {
10531    (usable && !has_unconfirmed_command(commands))
10532        .then(|| verdict::verified_noop(text))
10533        .flatten()
10534}
10535
10536fn short(commit: &str) -> String {
10537    commit.chars().take(7).collect()
10538}
10539
10540fn make_executable(path: &Path) -> Result<()> {
10541    #[cfg(unix)]
10542    {
10543        use std::os::unix::fs::PermissionsExt as _;
10544        let mut perms = std::fs::metadata(path)?.permissions();
10545        perms.set_mode(0o755);
10546        std::fs::set_permissions(path, perms)?;
10547    }
10548    #[cfg(not(unix))]
10549    {
10550        let _ = path;
10551    }
10552    Ok(())
10553}
10554
10555/// What every seat in one batch shares: where the answers are attributed, the
10556/// prompt overlay they inherit, and the build cache they are told to use.
10557///
10558/// A struct rather than four more parameters: `wave` also needs the run's
10559/// state (to record who is answering right now) and the attempt number, and
10560/// eight positional arguments is both unreadable and a clippy error.
10561struct WaveCtx<'a> {
10562    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10563    /// paid for it.
10564    run: &'a str,
10565    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10566    node: &'a str,
10567    prompts: &'a Prompts,
10568    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10569    cache: Option<&'a Path>,
10570    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10571    /// `JobRecord::round`. `None` for every other node.
10572    round: Option<usize>,
10573    /// Carry each seat's failed-agent history across waves (the review loop
10574    /// only): start-of-round priority and handover choice read
10575    /// [`RunState::seat_history`], and every answer or failure writes it.
10576    carry_seats: bool,
10577    /// The run's park / interrupt requests, so a wait for a shared resource
10578    /// can be abandoned the moment the operator asks to park.
10579    pauses: Option<(&'a Pause, &'a Pause)>,
10580}
10581
10582/// Shared scheduling policy for operations and spawned seat invocations.
10583fn repository_wait(
10584    state: &RunState,
10585    pauses: Option<(&Pause, &Pause)>,
10586) -> crate::repo_lock::WaitPolicy {
10587    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10588    let inherited = crate::repo_lock::WaitPolicy::current();
10589    crate::repo_lock::WaitPolicy::new(
10590        Duration::from_secs(state.config.graph.timeout_resource_wait),
10591        move || {
10592            inherited.cancelled()
10593                || pauses
10594                    .as_ref()
10595                    .is_some_and(|(a, b)| a.parked() || b.parked())
10596        },
10597    )
10598}
10599
10600fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10601    if error.is::<crate::repo_lock::ResourceBlocked>() {
10602        state.resource_block = Some(crate::run::ResourceBlock {
10603            cause: crate::run::ResourceBlockCause::Other,
10604            node: "repository".into(),
10605            kind: if error.to_string().contains("disk pressure") {
10606                "disk pressure"
10607            } else if error.to_string().contains("disk admission") {
10608                "disk admission"
10609            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10610                "incremental cache admission"
10611            } else {
10612                "repository coordination"
10613            }
10614            .into(),
10615            detail: error.to_string(),
10616            at: Timestamp::now(),
10617        });
10618        state.event("resource", error.to_string());
10619    }
10620}
10621
10622async fn coordinate_operation(
10623    state: &mut RunState,
10624    cwd: &Path,
10625    wait: &crate::repo_lock::WaitPolicy,
10626) -> Result<Option<crate::repo_lock::SeatLease>> {
10627    let result =
10628        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10629            .await;
10630    if let Err(e) = &result {
10631        record_repository_block(state, e);
10632    }
10633    result
10634}
10635
10636/// Run one job, honouring the parallelism budget.
10637async fn run_one(
10638    job: SeatJob,
10639    sem: Arc<Semaphore>,
10640    ctx: &WaveCtx<'_>,
10641    state: &mut RunState,
10642    attempt: usize,
10643) -> (SeatState, AgentOutcome) {
10644    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10645        .await
10646        .pop()
10647        .expect("one job in, one result out");
10648    (seat, out)
10649}
10650
10651/// Run every job concurrently, capped by the semaphore, preserving order.
10652///
10653/// Every seat in the batch is recorded into [`RunState::active`] before the
10654/// wave starts and cleared as each answer lands, so the run's own record says
10655/// who is still being waited on rather than only who finished.
10656async fn wave(
10657    jobs: Vec<SeatJob>,
10658    sem: Arc<Semaphore>,
10659    ctx: &WaveCtx<'_>,
10660    state: &mut RunState,
10661    attempt: usize,
10662) -> Vec<(usize, SeatState, AgentOutcome)> {
10663    let WaveCtx {
10664        run,
10665        node,
10666        prompts,
10667        cache,
10668        round,
10669        carry_seats: _,
10670        pauses,
10671    } = *ctx;
10672    for job in &jobs {
10673        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10674    }
10675    if let Err(e) = state.save() {
10676        // A failed persist of "who is answering right now" must not abort the
10677        // wave: the seats are already being asked, and the alternative is
10678        // losing the answers to save a status line nobody may even be
10679        // watching.
10680        tracing::warn!("could not persist in-progress seats: {e:#}");
10681    }
10682    // Hold the shared build cache's lease for the whole batch, not per job:
10683    // several candidates (an implement wave) or a fixer legitimately share
10684    // one cache concurrently within this run, and that stays untouched — a
10685    // single lease taken once for the whole wave and released once it is
10686    // done is what stops a *different* borrower (another run's own wave, its
10687    // e2e/gate, a human's `magi review`) from interleaving a build into the
10688    // same directory while this one is in flight. Best-effort, not
10689    // all-or-nothing: a wave that cannot get the lease within its own
10690    // longest job's budget still runs — an hour of paid implementer calls is
10691    // not thrown away over cache contention — but every write-allowed seat
10692    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10693    // below), the same fallback a read-only seat always gets, rather than
10694    // building into a directory this run was never granted. The identity
10695    // record is still invalidated below either way, so the next tracked
10696    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10697    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10698    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10699    // never borrowed from a job's agent budget (and never charged to it either,
10700    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10701    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10702    let cache_guard = if let Some(cache_dir) = cache {
10703        if jobs_had_a_writer {
10704            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10705            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10706            match acquire_cache_lease(
10707                state,
10708                cache_dir,
10709                &owner,
10710                resource_limit,
10711                node,
10712                node,
10713                &cancel,
10714            )
10715            .await
10716            {
10717                Ok(Some(g)) => Some(g),
10718                // Parking: leave without spawning anything. The run parks at
10719                // the next boundary and the seats are asked afresh on resume.
10720                Ok(None) => {
10721                    if let Some(block) = &mut state.resource_block {
10722                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10723                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10724                        };
10725                    }
10726                    return block_wave(
10727                        jobs,
10728                        state,
10729                        node,
10730                        attempt,
10731                        "parked while waiting for the shared build cache",
10732                    );
10733                }
10734                // An unreadable lease is not contention: keep the old
10735                // fallback (no shared cache for this wave's seats).
10736                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10737                Err(e) => {
10738                    // Writing seats may not build into a cache this run does
10739                    // not hold, and an agent that cannot be given one is not
10740                    // asked at all: the run stops resumably instead.
10741                    let why = format!("{e:#}");
10742                    state.event(
10743                        "resource",
10744                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10745                    );
10746                    return block_wave(jobs, state, node, attempt, &why);
10747                }
10748            }
10749        } else {
10750            None
10751        }
10752    } else {
10753        None
10754    };
10755    // Every job keeps its whole budget: the wait above was not the agent's.
10756    // Restart the seats' clocks so "started" means "being spawned now".
10757    state.seats_spawning(node, attempt);
10758    if let Err(e) = state.save() {
10759        tracing::warn!("could not persist the start of a wave: {e:#}");
10760    }
10761    // Queueing behind this wave's own seats for a parallelism slot is normal
10762    // (each of them is bounded by its own agent timeout); only a wait beyond
10763    // all of them plus the resource limit means the slots never came back.
10764    let slot_limit = resource_limit
10765        + jobs
10766            .iter()
10767            .map(|j| j.timeout)
10768            .max()
10769            .unwrap_or_default()
10770            .saturating_mul(jobs.len() as u32);
10771    let mut set = tokio::task::JoinSet::new();
10772    // Seats announce themselves here the moment they hold a parallelism
10773    // permit, i.e. just before their CLI is spawned.
10774    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10775    let overlay = prompts.overlay(node);
10776    let repository_wait = repository_wait(state, pauses);
10777    for (i, mut job) in jobs.into_iter().enumerate() {
10778        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10779        if cache.is_some() {
10780            job.prompt.push('\n');
10781            job.prompt
10782                .push_str(&prompt::build_cache_note(node, job.allow_write));
10783        }
10784        let sem = Arc::clone(&sem);
10785        let run = run.to_owned();
10786        let node = node.to_owned();
10787        // Only implementers were told about the task's attachments, so only
10788        // their seats get the directory widened for reading.
10789        let attachments = if node == "implement" {
10790            state.attachments.clone()
10791        } else {
10792            Vec::new()
10793        };
10794        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10795        // `prompt::build_cache_note`'s doc for why setting it anyway is
10796        // exactly how a sandboxed reviewer's write refusal got reported as a
10797        // defect in the patch, not a property of its own seat. And a
10798        // write-allowed one is handed it only when the lease above was
10799        // actually acquired: a wave that could not get it (`cache_guard` is
10800        // `None`, see its own comment) must not send seats to build into a
10801        // directory this run does not hold - that is the exact concurrent,
10802        // unmanaged-write race this module exists to prevent, not something
10803        // "proceeding anyway" is allowed to reintroduce.
10804        let cache = cache
10805            .filter(|_| job.allow_write && cache_guard.is_some())
10806            .map(Path::to_path_buf);
10807        let spawned_tx = spawned_tx.clone();
10808        let repository_wait = repository_wait.clone();
10809        let coordination = state.config.coordination.clone();
10810        let fetched_base = state.fetched_base.clone();
10811        set.spawn(async move {
10812            // A wait for a parallelism slot is scheduling, not the agent: it
10813            // is bounded by its own limit and ends as a resource block, never
10814            // as a timeout of the agent that was not started.
10815            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10816                Ok(p) => p,
10817                Err(_) => {
10818                    return (
10819                        i,
10820                        job.seat,
10821                        AgentOutcome::ResourceBlocked(format!(
10822                            "no parallel slot became free within {}s",
10823                            slot_limit.as_secs()
10824                        )),
10825                        false,
10826                    );
10827                }
10828            };
10829            spawned_tx.send(job.seat.key.clone()).ok();
10830            let mut seat = job.seat;
10831            let out = repository_wait
10832                .scope(agent::invoke(
10833                    &job.spec,
10834                    &mut seat,
10835                    &Invocation {
10836                        coordination: coordination.as_ref(),
10837                        fetched_base: fetched_base.as_ref(),
10838                        cwd: &job.cwd,
10839                        prompt: &job.prompt,
10840                        timeout: job.timeout,
10841                        allow_write: job.allow_write,
10842                        sessions: job.sessions,
10843                        artifacts: &job.artifacts,
10844                        stem: &job.stem,
10845                        run: &run,
10846                        node: &node,
10847                        cache_dir: cache.as_deref(),
10848                        attachments: &attachments,
10849                        writable: &[],
10850                    },
10851                ))
10852                .await;
10853            let interrupted = out
10854                .as_ref()
10855                .err()
10856                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10857            let out = match out {
10858                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10859                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10860                // Billed work the CLI failed to hand over is not an ordinary
10861                // failure, but its text is the CLI's raw error JSON, not an
10862                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10863                // read it as one by forgetting to check. `usable()` is always
10864                // false here (dropped implies an empty response), so this has
10865                // to be checked before the catch-all `Failed` below or the
10866                // one shape this exists for is lost with the rest.
10867                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10868                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10869                Ok(o) => AgentOutcome::Failed(format!(
10870                    "process exited with no usable output ({})",
10871                    crate::notices::exit_status(o.exit_code)
10872                )),
10873                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10874                    AgentOutcome::ResourceBlocked(e.to_string())
10875                }
10876                Err(e) => AgentOutcome::Failed(e.to_string()),
10877            };
10878            (i, seat, out, interrupted)
10879        });
10880    }
10881    drop(spawned_tx);
10882    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10883    loop {
10884        let joined = tokio::select! {
10885            Some(key) = spawned_rx.recv() => {
10886                // The clock of this seat starts now, not when it queued.
10887                state.seat_spawned(&key);
10888                if let Err(e) = state.save() {
10889                    tracing::warn!("could not persist a seat's start: {e:#}");
10890                }
10891                continue;
10892            }
10893            joined = set.join_next() => joined,
10894        };
10895        let Some(joined) = joined else {
10896            break;
10897        };
10898        let (i, seat, out, interrupted) = match joined {
10899            Ok(v) => v,
10900            // No seat to clear: a panicked task never reported which one it
10901            // was. The defensive sweep below this loop is what stops that
10902            // seat's `active` entry from surviving forever.
10903            Err(e) => {
10904                tracing::error!("agent task panicked: {e}");
10905                continue;
10906            }
10907        };
10908        state.seat_finished(&seat.key);
10909        if interrupted
10910            && node == "implement"
10911            && let Some(label) = state
10912                .candidates
10913                .iter()
10914                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10915                .map(|candidate| candidate.label)
10916        {
10917            state.candidate_preservation.insert(
10918                label,
10919                "interrupted implementation awaiting persistence".into(),
10920            );
10921            state.interrupted_implementations.insert(label);
10922        }
10923        if let AgentOutcome::ResourceBlocked(why) = &out
10924            && state.resource_block.is_none()
10925        {
10926            state.resource_block = Some(crate::run::ResourceBlock {
10927                cause: crate::run::ResourceBlockCause::Other,
10928                node: node.to_owned(),
10929                kind: if why.contains("disk pressure") {
10930                    "disk pressure"
10931                } else if why.contains("disk admission") {
10932                    "disk admission"
10933                } else if why.contains(crate::incremental::BLOCKED) {
10934                    "incremental cache admission"
10935                } else if why.contains("repository coordination") {
10936                    "repository coordination"
10937                } else {
10938                    "parallel slot"
10939                }
10940                .to_owned(),
10941                detail: why.clone(),
10942                at: Timestamp::now(),
10943            });
10944            let activity = if interrupted {
10945                "agent execution interrupted"
10946            } else {
10947                "no agent was started"
10948            };
10949            state.event("resource", format!("{node}: {activity} - {why}"));
10950        }
10951        record_jobs(state, node, round, &seat.key, &out);
10952        if let Err(e) = state.save() {
10953            tracing::warn!("could not persist a seat's completion: {e:#}");
10954        }
10955        if collected.len() <= i {
10956            collected.resize_with(i + 1, || None);
10957        }
10958        collected[i] = Some((i, seat, out));
10959    }
10960    // Belt-and-braces for the panic branch above: every seat this exact batch
10961    // started shares this `(node, attempt)` pair, and every seat that finished
10962    // normally already cleared itself, so anything left tagged with it here
10963    // can only be a panicked task's leftover. Cleared unconditionally rather
10964    // than left to read as still answering forever.
10965    if state
10966        .active
10967        .values()
10968        .any(|a| a.node == node && a.attempt == attempt)
10969    {
10970        state
10971            .active
10972            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10973        if let Err(e) = state.save() {
10974            tracing::warn!("could not persist the end of a wave: {e:#}");
10975        }
10976    }
10977    // Whether or not the lease above was actually held, several worktrees
10978    // may just have built into the cache with nothing here able to name one
10979    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10980    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10981    // clean it might not have strictly needed; trusting a stale match would
10982    // cost it a wrong answer.
10983    if let Some(cache_dir) = cache
10984        && jobs_had_a_writer
10985    {
10986        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10987    }
10988    if let Some(guard) = cache_guard {
10989        guard.release();
10990    }
10991    collected.into_iter().flatten().collect()
10992}
10993
10994/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10995/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10996/// this is data collection, not the fix-specific completion contract in
10997/// [`Runner::continue_fix_report`], and applies regardless of which node
10998/// asked.
10999///
11000/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11001/// evidence from; `Failed` does not, and correctly contributes nothing — a
11002/// timeout or crash is not itself evidence about a command the seat may have
11003/// started.
11004fn record_jobs(
11005    state: &mut RunState,
11006    node: &str,
11007    round: Option<usize>,
11008    seat: &str,
11009    out: &AgentOutcome,
11010) {
11011    let commands: &[agent::CommandEvidence] = match out {
11012        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11013        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11014    };
11015    let checked_at = Timestamp::now();
11016    for c in commands {
11017        state.jobs.push(JobRecord {
11018            node: node.to_owned(),
11019            round,
11020            seat: seat.to_owned(),
11021            id: c.id.clone(),
11022            description: c.description.clone(),
11023            checked_at,
11024            status: match c.exit_code {
11025                Some(0) => JobStatus::Completed,
11026                Some(_) => JobStatus::Failed,
11027                None => JobStatus::Unknown,
11028            },
11029            exit_code: c.exit_code,
11030            result_summary: c.result_summary.clone(),
11031            source: c.source.clone(),
11032        });
11033    }
11034}
11035
11036/// Is a review round clean, given how many reviewer seats answered against
11037/// how many the round expected?
11038///
11039/// A seat that never answered (timeout, crash, unparsable output) is not a
11040/// seat that read the patch and found nothing — treating it as such is
11041/// exactly the bug this function exists to close. Under the default `block`
11042/// policy a missing seat can never be clean; `warn` still requires the seats
11043/// that *did* answer to have found nothing blocking and verification to be
11044/// green.
11045///
11046/// `quota_missing` narrows that `block` default for exactly one cause of
11047/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11048/// a session limit lifts by the very next round buys nothing — the seat is
11049/// asked again with the same quota — so once every missing seat is accounted
11050/// for by a quota loss (and at least one seat *did* answer, so a decision has
11051/// something to rest on) the round is decided on the panel that could answer,
11052/// same as `warn` would. A panel that lost every seat to quota is not
11053/// decided here: `answered == 0` falls through to the existing `block`
11054/// fallback so a fully collapsed panel still waits rather than landing on no
11055/// review at all.
11056fn round_is_clean(
11057    blocking: usize,
11058    e2e_ok: bool,
11059    answered: usize,
11060    expected: usize,
11061    quota_missing: usize,
11062    policy: IncompleteReviewPolicy,
11063) -> bool {
11064    if blocking != 0 || !e2e_ok {
11065        return false;
11066    }
11067    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11068        return true;
11069    }
11070    answered > 0 && expected - answered <= quota_missing
11071}
11072
11073/// The review loop's own conclusion, derived entirely from its persisted
11074/// round records and the round budget that produced them — never from
11075/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11076/// recomputes the identical answer regardless of what an earlier node in the
11077/// same walk, or a previous walk, did to `status`.
11078///
11079/// `None` while more rounds remain to try, including when review never ran
11080/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11081/// gone clean, or the budget is spent, or the tree has stopped moving (see
11082/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11083///
11084/// - An incomplete panel that raised nothing is missing input, not a
11085///   verified tree — never a hand-off candidate, whatever verification said
11086///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11087/// - Otherwise, green e2e on the last round hands off (see
11088///   [`Runner::stop_reviewing`]); red e2e blocks.
11089///
11090/// A last round whose own verification is still `ResourceBlocked` — magi
11091/// itself never got a command to run, not evidence the patch is broken —
11092/// is neither: this returns `None` for it too, the same as "more rounds
11093/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11094/// handling of that shape) instead of this cheap recomputation guessing a
11095/// verdict a real attempt never produced.
11096pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11097    if max_rounds == 0
11098        || reviews.last().is_some_and(|r| {
11099            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11100        })
11101    {
11102        return Some(RunStatus::Gating);
11103    }
11104    let last = reviews.last()?;
11105    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11106    if reviews.len() < max_rounds && !stagnant {
11107        return None;
11108    }
11109    if last.incomplete() && last.blocking == 0 {
11110        return Some(RunStatus::Blocked);
11111    }
11112    if last.e2e_status() == E2eStatus::ResourceBlocked
11113        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11114    {
11115        return None;
11116    }
11117    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11118        RunStatus::Gating
11119    } else {
11120        RunStatus::Blocked
11121    })
11122}
11123
11124/// How long a re-ask may take, given the budget the first attempt had.
11125///
11126/// A `nudged` retry is a request to restate an answer the seat has already
11127/// worked out: it carries no new work, so it does not deserve the original
11128/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11129/// 133 seconds while a third sat for over ten minutes on a resumed session
11130/// holding 410 KB of prior output - and because the retry had inherited the
11131/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11132/// judging round whose other seats were long finished.
11133///
11134/// A quarter of the budget, with a floor so that a deliberately short timeout
11135/// does not collapse to nothing. A retry that re-sends the whole prompt
11136/// (because the seat kept no context) is the original job again, and keeps the
11137/// original budget.
11138pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11139    if nudged {
11140        (full / 4).max(Duration::from_secs(120)).min(full)
11141    } else {
11142        full
11143    }
11144}
11145
11146/// Run a wave and parse each reply, re-asking the seats whose reply was
11147/// unusable.
11148///
11149/// The re-ask is a nudge rather than the whole prompt again when the seat still
11150/// holds its conversation, which is the difference between a cheap retry and
11151/// paying for the entire candidate set twice.
11152///
11153/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11154/// successor in `roster` is handed to it instead of being re-asked: the
11155/// handover is the retry. A seat with no successor left (a single-agent
11156/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11157/// the asks to one seat in one node number at most
11158/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11159/// is asked once (a dropped stream is nudged first), and only the last agent
11160/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11161/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11162/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11163/// other failure stops the chain when the previous agent failed the same way
11164/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11165/// [`has_context`] is false and the job's own full prompt and full budget are
11166/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11167/// intermediate ones are not losses) and is returned as a failure like any
11168/// other absent seat — the caller decides whether the panel still has a
11169/// quorum. An empty `roster` disables handover: failures are nudged as they
11170/// always were, and a quota is simply lost. A reply that fails to parse or
11171/// validate is the prompt's doing and is only ever nudged, never handed over.
11172///
11173/// The returned [`SeatState`] names the agent that answered (or tried last).
11174#[allow(clippy::too_many_arguments)]
11175async fn ask_json_wave<T>(
11176    jobs: Vec<SeatJob>,
11177    sem: Arc<Semaphore>,
11178    retries: usize,
11179    roster: &[AgentSpec],
11180    ctx: &WaveCtx<'_>,
11181    losses: &mut Vec<QuotaLoss>,
11182    state: &mut RunState,
11183    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11184) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11185where
11186    T: serde::de::DeserializeOwned + Send + 'static,
11187{
11188    ask_wave_with(
11189        jobs,
11190        sem,
11191        retries,
11192        roster,
11193        ctx,
11194        losses,
11195        state,
11196        &|text: &str| {
11197            let v = verdict::extract_json::<T>(text)?;
11198            validate(&v)?;
11199            Ok(v)
11200        },
11201    )
11202    .await
11203}
11204
11205/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11206/// a node whose answer is prose (deliberation) shares the same handover,
11207/// failure classification, quota bookkeeping and bounds instead of a copy.
11208///
11209/// A seat handed to another roster agent is sent the job's `handover` prompt
11210/// (when it has one) rather than `prompt`: the new agent has no session, so a
11211/// resume-style prompt would be incomplete. That holds for the handover ask
11212/// and for every nudge to that agent whose `has_context` is false.
11213#[allow(clippy::too_many_arguments)]
11214async fn ask_wave_with<T>(
11215    jobs: Vec<SeatJob>,
11216    sem: Arc<Semaphore>,
11217    retries: usize,
11218    roster: &[AgentSpec],
11219    ctx: &WaveCtx<'_>,
11220    losses: &mut Vec<QuotaLoss>,
11221    state: &mut RunState,
11222    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11223) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11224where
11225    T: Send + 'static,
11226{
11227    let n = jobs.len();
11228    let originals: Vec<SeatJob> = jobs;
11229    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11230    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11231    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11232    // answer, N once it has gone through N nudges. Read back once this
11233    // returns, so a caller building a history record (`ReviewRecord`) can
11234    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11235    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11236    // field's own doc.
11237    let mut nudges: Vec<usize> = vec![0; n];
11238    // The agent now occupying each seat, the ids it has already been through,
11239    // where in the roster the walk began, the class of the last failure, and
11240    // the stem word of a handover not yet asked (full prompt, full budget).
11241    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11242    let mut tried: Vec<BTreeSet<String>> = specs
11243        .iter()
11244        .map(|s| BTreeSet::from([s.id.clone()]))
11245        .collect();
11246    let starts: Vec<usize> = specs
11247        .iter()
11248        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11249        .collect();
11250    let carry = ctx.carry_seats && !roster.is_empty();
11251    // Carried across rounds: ids that failed the seat earlier, and how the
11252    // last failure went (so a repeat of it is not handed over again).
11253    let carried: Vec<BTreeSet<String>> = originals
11254        .iter()
11255        .map(|j| {
11256            state
11257                .seat_history
11258                .get(&j.seat.key)
11259                .filter(|_| carry)
11260                .map(|h| h.failed.clone())
11261                .unwrap_or_default()
11262        })
11263        .collect();
11264    let mut prev: Vec<Option<FailClass>> = originals
11265        .iter()
11266        .map(|j| {
11267            state
11268                .seat_history
11269                .get(&j.seat.key)
11270                .filter(|_| carry)
11271                .and_then(|h| h.last_fail.clone())
11272        })
11273        .collect();
11274    let next_agent =
11275        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11276            let others: BTreeSet<String> = specs
11277                .iter()
11278                .enumerate()
11279                .filter(|(j, _)| *j != i)
11280                .map(|(_, s)| s.id.clone())
11281                .collect();
11282            pick_successor(
11283                roster,
11284                starts[i],
11285                tried,
11286                carry.then(|| &carried[i]),
11287                &others,
11288            )
11289            .cloned()
11290        };
11291    let mut fresh: Vec<Option<String>> = vec![None; n];
11292    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11293    let mut pending: Vec<usize> = (0..n).collect();
11294
11295    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11296    // asks: an agent with a successor is asked once and handed over, and only
11297    // a seat with no successor spends `retries` nudges on the same agent. This
11298    // only guarantees the loop's own termination whatever those say.
11299    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11300    for round in 0..max_rounds {
11301        if pending.is_empty() {
11302            break;
11303        }
11304        let mut batch = Vec::with_capacity(pending.len());
11305        let mut renudged: Vec<&str> = Vec::new();
11306        for &i in &pending {
11307            let src = &originals[i];
11308            // A seat now held by another agent than the job named has no
11309            // session of its own: it gets the full-context prompt whenever
11310            // it is asked in full (the handover ask, a nudge it cannot
11311            // resume).
11312            let full: &str = match &src.handover {
11313                Some(h) if specs[i].id != src.spec.id => h,
11314                _ => &src.prompt,
11315            };
11316            // The prompt and the budget are one decision: a nudge restates
11317            // finished work, a re-sent prompt redoes it.
11318            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11319                (
11320                    full.to_owned(),
11321                    src.timeout,
11322                    format!("{}-{word}-{}", src.stem, specs[i].id),
11323                )
11324            } else if nudges[i] == 0 {
11325                (full.to_owned(), src.timeout, src.stem.clone())
11326            } else {
11327                renudged.push(src.seat.key.as_str());
11328                let why = done[i]
11329                    .as_ref()
11330                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11331                    .unwrap_or_else(|| "no parsable answer".to_owned());
11332                let nudge = prompt::nudge(&why);
11333                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11334                let prompt = if nudged {
11335                    nudge
11336                } else {
11337                    format!("{full}\n\n---\n\n{nudge}")
11338                };
11339                (
11340                    prompt,
11341                    retry_budget(src.timeout, nudged),
11342                    format!("{}-retry{}", src.stem, nudges[i]),
11343                )
11344            };
11345            batch.push(SeatJob {
11346                spec: specs[i].clone(),
11347                seat: seats[i].clone(),
11348                cwd: src.cwd.clone(),
11349                prompt,
11350                timeout,
11351                allow_write: src.allow_write,
11352                sessions: src.sessions,
11353                artifacts: src.artifacts.clone(),
11354                stem,
11355                handover: None,
11356            });
11357        }
11358
11359        if !renudged.is_empty() {
11360            state.event(
11361                ctx.node,
11362                format!("retry {round}: re-asking {}", renudged.join(", ")),
11363            );
11364        }
11365        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11366        let mut still = Vec::new();
11367        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11368            seats[i] = seat;
11369            // No agent was asked: nothing to hand over, nudge, remember or
11370            // charge. The caller sees `resource_block` and stops the node.
11371            if let AgentOutcome::ResourceBlocked(why) = &out {
11372                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11373                continue;
11374            }
11375            let class = FailClass::of(&out);
11376            // A dropped stream is nudged first (the conversation is still
11377            // there to pick up); only a seat whose nudges are spent hands over.
11378            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11379                && nudges[i] < retries
11380                && !roster.is_empty();
11381            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11382                let next = should_hand_over(prev[i].as_ref(), &cur)
11383                    .then(|| next_agent(i, &tried[i], &specs))
11384                    .flatten();
11385                if carry {
11386                    let h = state
11387                        .seat_history
11388                        .entry(originals[i].seat.key.clone())
11389                        .or_default();
11390                    h.failed.insert(specs[i].id.clone());
11391                    h.last_fail = Some(cur.clone());
11392                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11393                        h.last_ok = None;
11394                    }
11395                    // Saved before the next agent is asked, so a restart in
11396                    // between does not forget who failed.
11397                    if let Err(e) = state.save() {
11398                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11399                    }
11400                }
11401                if let Some(next) = next {
11402                    record_handover(
11403                        state,
11404                        ctx.node,
11405                        &originals[i].seat.key,
11406                        &specs[i].id,
11407                        &next.id,
11408                        &cur,
11409                        &fail_reason(&out),
11410                    );
11411                    tried[i].insert(next.id.clone());
11412                    prev[i] = Some(cur.clone());
11413                    seats[i] =
11414                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11415                    specs[i] = next;
11416                    fresh[i] = Some(cur.stem_word().to_owned());
11417                    nudges[i] = 0;
11418                    done[i] = Some(Err(anyhow::anyhow!(
11419                        "handed over after: {}",
11420                        fail_reason(&out)
11421                    )));
11422                    still.push(i);
11423                    continue;
11424                }
11425            }
11426            if carry
11427                && !nudge_first
11428                && let Some(cur) = class.clone()
11429            {
11430                // The chain ended here (no successor, or a repeated failure).
11431                let h = state
11432                    .seat_history
11433                    .entry(originals[i].seat.key.clone())
11434                    .or_default();
11435                h.failed.insert(specs[i].id.clone());
11436                h.last_fail = Some(cur);
11437            }
11438            let parsed = match out {
11439                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11440                AgentOutcome::Quota(o) => {
11441                    last_quota[i] = o.quota.clone();
11442                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11443                }
11444                // Not a parseable answer: the nudge loop re-asks it, which is
11445                // exactly what a dropped stream needs. Just don't hand its raw
11446                // error JSON to `extract_json`.
11447                AgentOutcome::Dropped(o) => {
11448                    let why = o
11449                        .dropped
11450                        .as_ref()
11451                        .map(|d| d.why.as_str())
11452                        .unwrap_or("the CLI ended the stream without delivering its answer");
11453                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11454                }
11455                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11456                    Err(anyhow::anyhow!(e))
11457                }
11458            };
11459            let quota = class == Some(FailClass::Quota);
11460            let failed = parsed.is_err();
11461            if !failed && matches!(ctx.node, "judge" | "vote") {
11462                // A validated answer restores this seat's quorum presence,
11463                // including on the normal path after an all-seat block.
11464                state
11465                    .quota
11466                    .retain(|loss| loss.seat != originals[i].seat.key);
11467            }
11468            done[i] = Some(parsed);
11469            if carry && !failed {
11470                let h = state
11471                    .seat_history
11472                    .entry(originals[i].seat.key.clone())
11473                    .or_default();
11474                h.failed.remove(&specs[i].id);
11475                h.last_ok = Some(specs[i].id.clone());
11476                h.last_fail = None;
11477            }
11478            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11479            // fail the same way; and never re-ask a seat that already parsed.
11480            // A failed agent that still has a successor is not re-asked
11481            // either: the handover was its remedy and has just been refused
11482            // (the chain stops on a repeated failure class). A seat with no
11483            // successor left (a single-agent roster, the roster's tail, or an
11484            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11485            let agent_failure = class.is_some()
11486                && !nudge_first
11487                && !roster.is_empty()
11488                && next_agent(i, &tried[i], &specs).is_some();
11489            if failed && !quota && !agent_failure && nudges[i] < retries {
11490                nudges[i] += 1;
11491                still.push(i);
11492            }
11493        }
11494        pending = still;
11495    }
11496
11497    // One loss per seat whose chain ended on a quota: the intermediate ones
11498    // were absorbed by a handover and are not losses.
11499    for (i, q) in last_quota.into_iter().enumerate() {
11500        if let Some(quota) = q {
11501            losses.push(QuotaLoss::from_report(
11502                originals[i].seat.key.clone(),
11503                ctx.node,
11504                Some(&quota),
11505            ));
11506        }
11507    }
11508
11509    // A stalled quorum already has its own recovery checkpoint. Refresh
11510    // its missing seats before deciding whether it can resume.
11511    if state.status != RunStatus::Stalled
11512        && ctx.node != "deliberate"
11513        && n > 0
11514        && losses.len() == n
11515        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11516    {
11517        block_machine_outage(state, ctx.node, losses);
11518        state.quota.append(losses);
11519    }
11520
11521    seats
11522        .into_iter()
11523        .zip(done)
11524        .zip(nudges)
11525        .map(|((seat, res), attempts)| {
11526            (
11527                seat,
11528                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11529                attempts,
11530            )
11531        })
11532        .collect()
11533}
11534
11535/// A node with no available agent has no task verdict. Preserve the node
11536/// checkpoint and record why a subsequent explicit resume is needed.
11537pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11538    let mut auth: Vec<_> = losses
11539        .iter()
11540        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11541        .map(|loss| loss.agent.clone())
11542        .collect();
11543    auth.sort();
11544    auth.dedup();
11545    let (kind, detail) = if auth.is_empty() {
11546        (
11547            "agent usage limit",
11548            "All seats reached their agent usage limits; resume after usage is available"
11549                .to_owned(),
11550        )
11551    } else {
11552        (
11553            "agent authentication",
11554            format!(
11555                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11556                auth.join(", ")
11557            ),
11558        )
11559    };
11560    state.resource_block = Some(crate::run::ResourceBlock {
11561        node: node.to_owned(),
11562        kind: kind.to_owned(),
11563        cause: crate::run::ResourceBlockCause::Other,
11564        detail: detail.clone(),
11565        at: Timestamp::now(),
11566    });
11567    state.event(node, detail);
11568}
11569
11570/// Did the wave end without asking any agent (see [`block_wave`])?
11571fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11572    results
11573        .iter()
11574        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11575}
11576
11577/// Settle a wave that never spawned anything: every job answers
11578/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11579/// else about the seat (session, history, quota) is touched.
11580fn block_wave(
11581    jobs: Vec<SeatJob>,
11582    state: &mut RunState,
11583    node: &str,
11584    attempt: usize,
11585    why: &str,
11586) -> Vec<(usize, SeatState, AgentOutcome)> {
11587    state.resource_wait = None;
11588    state
11589        .active
11590        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11591    if let Err(e) = state.save() {
11592        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11593    }
11594    jobs.into_iter()
11595        .enumerate()
11596        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11597        .collect()
11598}
11599
11600/// Acquire the shared build cache's lease, waiting out contention within
11601/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11602/// unbounded wait is never acceptable).
11603///
11604/// A first, non-blocking check happens before ever waiting; if it finds the
11605/// lease busy, that fact is logged as a `verify` event *and* flushed with
11606/// [`RunState::save`] immediately — not only once the wait finally succeeds
11607/// or gives up — so a `magi show` run by a different process while this one
11608/// is still waiting reads a `run.json` that says so, rather than whatever it
11609/// looked like before the wait started. The same applies to the terminal
11610/// failure: logged and saved before this returns `Err`, so a caller that
11611/// could not get the lease at all still leaves a legible record of why.
11612async fn acquire_cache_lease(
11613    state: &mut RunState,
11614    cache_dir: &Path,
11615    owner: &crate::cache::Owner,
11616    budget: Duration,
11617    node: &str,
11618    context: &str,
11619    cancel: &(dyn Fn() -> bool + Sync),
11620) -> Result<Option<crate::cache::Guard>> {
11621    let home = crate::run::home();
11622    let started = Instant::now();
11623    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11624        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11625        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11626        Err(e) => {
11627            state.event(
11628                "verify",
11629                format!("{context}: could not check the shared build cache: {e:#}"),
11630            );
11631            if let Err(e2) = state.save() {
11632                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11633            }
11634            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11635        }
11636    };
11637    state.event(
11638        "verify",
11639        format!(
11640            "{context}: waiting for the shared build cache at {} ({})",
11641            cache_dir.display(),
11642            busy.describe()
11643        ),
11644    );
11645    // Seats of this node are waiting, not running: say so, so the countdown a
11646    // reader sees only ever measures an agent that has been spawned.
11647    state.resource_wait = Some(crate::run::ResourceWait {
11648        node: context.to_owned(),
11649        kind: "shared build cache".to_owned(),
11650        holder: busy.describe(),
11651        since: Timestamp::now(),
11652        limit_secs: budget.as_secs(),
11653    });
11654    if let Err(e) = state.save() {
11655        tracing::warn!("could not persist a cache wait: {e:#}");
11656    }
11657    let remaining = budget.saturating_sub(started.elapsed());
11658    let waited = crate::cache::wait_for_unless(
11659        &home,
11660        cache_dir,
11661        owner,
11662        remaining,
11663        Duration::from_secs(5),
11664        cancel,
11665    )
11666    .await;
11667    state.resource_wait = None;
11668    let cause = match &waited {
11669        Ok(Some(_)) => None,
11670        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11671        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11672            Some(crate::run::ResourceBlockCause::Timeout {
11673                waited_secs: started.elapsed().as_secs(),
11674                limit_secs: budget.as_secs(),
11675            })
11676        }
11677        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11678    };
11679    if let Some(cause) = cause {
11680        let detail = match &waited {
11681            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11682            Err(error) => format!("{error:#}"),
11683        };
11684        state.resource_block = Some(crate::run::ResourceBlock {
11685            cause,
11686            node: node.to_owned(),
11687            kind: "shared build cache".into(),
11688            detail,
11689            at: Timestamp::now(),
11690        });
11691        state.event(
11692            "resource",
11693            state.resource_block.as_ref().unwrap().description(),
11694        );
11695    }
11696    if let Err(error) = state.save() {
11697        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11698    }
11699    waited
11700}
11701
11702/// The lease state could not be read at all (unreadable lease file, a
11703/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11704/// known to be contended, so a wave degrades to running without a shared cache
11705/// instead of stopping on a hold that [`crate::triage`] could never release.
11706#[derive(Debug)]
11707struct CacheCheckError(String);
11708
11709impl std::fmt::Display for CacheCheckError {
11710    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11711        f.write_str(&self.0)
11712    }
11713}
11714
11715impl std::error::Error for CacheCheckError {}
11716
11717/// Keep a specific acquisition outcome when the caller adds command evidence.
11718fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11719    state
11720        .resource_block
11721        .get_or_insert_with(|| crate::run::ResourceBlock {
11722            cause: crate::run::ResourceBlockCause::Other,
11723            node: node.to_owned(),
11724            kind: "shared build cache".into(),
11725            detail: format!("{error:#}"),
11726            at: Timestamp::now(),
11727        });
11728    if let Err(error) = state.save() {
11729        tracing::warn!("could not persist cache resource block: {error:#}");
11730    }
11731}
11732
11733/// Run `body` — a verify command batch — while holding the shared build
11734/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11735/// never interleave with another borrower's build against the same
11736/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11737/// timeout, or a human's own `magi review`. See the `cache` module doc for
11738/// why this matters more than Cargo's own per-target locking covers — two
11739/// *different* worktrees building the same package name/version into one
11740/// cache directory is a staleness bug, not a lock contention one.
11741///
11742/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11743/// agent waves do. Repository scheduling also has its own bounded,
11744/// cancellable resource-wait policy. Contention must not consume the
11745/// verification budget: `body` receives the full `budget` after both leases
11746/// are acquired and the cache is fresh. Without a shared cache, only
11747/// repository coordination is acquired before handing over the full budget.
11748///
11749/// A lease that cannot be acquired within the resource wait is reported as a single
11750/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11751/// verification — the same shape a spawn failure already takes in
11752/// [`run_commands`], so a caller need not special-case it.
11753#[allow(clippy::too_many_arguments)]
11754async fn with_cache_lease<'s, F, Fut>(
11755    state: &'s mut RunState,
11756    cache_dir: Option<&Path>,
11757    node: &str,
11758    seat: &str,
11759    worktree: &Path,
11760    head: &str,
11761    budget: Duration,
11762    context: &str,
11763    wait: &crate::repo_lock::WaitPolicy,
11764    body: F,
11765) -> Result<(Vec<CommandOutcome>, bool)>
11766where
11767    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11768    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11769{
11770    // Match editing waves: cache first, repository second. In particular,
11771    // pre_gate must never hold the repository while waiting for a wave's cache.
11772    let started = Instant::now();
11773    let home = crate::run::home();
11774    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11775    let guard = if let Some(cache_dir) = cache_dir {
11776        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11777        match acquire_cache_lease(
11778            state,
11779            cache_dir,
11780            &owner,
11781            resource_limit,
11782            node,
11783            context,
11784            &|| wait.cancelled(),
11785        )
11786        .await
11787        .and_then(|g| {
11788            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11789        }) {
11790            Ok(g) => Some(g),
11791            Err(e) => {
11792                record_cache_error(state, node, &e);
11793                return Ok((
11794                    vec![CommandOutcome {
11795                        command: "(waiting for the shared build cache)".into(),
11796                        code: None,
11797                        output_tail: e.to_string(),
11798                        duration_ms: started.elapsed().as_millis() as u64,
11799                        resource_blocked: true,
11800                    }],
11801                    false,
11802                ));
11803            }
11804        }
11805    } else {
11806        None
11807    };
11808    let _coordination = coordinate_operation(state, worktree, wait).await?;
11809    if let Some(cache_dir) = cache_dir {
11810        let identity = crate::cache::Identity::new(worktree, head);
11811        if let Err(e) = crate::cache::ensure_fresh(
11812            &home,
11813            cache_dir,
11814            &identity,
11815            &state.config.verify.cargo_manifests(),
11816        ) {
11817            record_cache_error(state, node, &e);
11818            state.event(
11819                "verify",
11820                format!(
11821                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11822                    worktree.display(),
11823                    short(head)
11824                ),
11825            );
11826            if let Some(guard) = guard {
11827                guard.release();
11828            }
11829            return Ok((
11830                vec![CommandOutcome {
11831                    command: "(confirming the shared build cache is fresh)".into(),
11832                    code: None,
11833                    output_tail: e.to_string(),
11834                    duration_ms: started.elapsed().as_millis() as u64,
11835                    resource_blocked: true,
11836                }],
11837                false,
11838            ));
11839        }
11840    }
11841    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11842    if !timed_out_pids.is_empty() {
11843        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11844    }
11845    // Drop repository coordination before releasing the cache, reversing the
11846    // acquisition order. A cancelled/error future also drops both guards.
11847    drop(_coordination);
11848    if let Some(guard) = guard {
11849        guard.release();
11850    }
11851    Ok((outcomes, retried))
11852}
11853
11854/// Poll `pids` — commands [`run_commands`] reports as still running when its
11855/// own timeout elapsed — until every one is confirmed gone, or
11856/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11857///
11858/// Real confirmation where confirmation is possible, not a substitute for
11859/// full process-tree observation: a grandchild the timed-out process spawned
11860/// and that survives independently of it is invisible to a pid check the
11861/// same way it always was, and continuing to observe and collect *that*
11862/// stays a different piece of work with its own owner. This only narrows a
11863/// fixed blind wait into an actual check of the pids this process does know
11864/// about.
11865async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11866    wait_for_pids_with(
11867        pids,
11868        crate::proc::pid_alive,
11869        LEASE_RELEASE_POLL,
11870        LEASE_RELEASE_MAX_WAIT,
11871    )
11872    .await;
11873}
11874
11875/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11876/// interval and ceiling supplied by the caller, so the polling *logic* -
11877/// returns as soon as every pid reports dead, gives up at the ceiling
11878/// otherwise - is testable on millisecond durations without asking the real
11879/// OS about a pid at all.
11880async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11881    pids: &[u32],
11882    alive: F,
11883    poll: Duration,
11884    max_wait: Duration,
11885) {
11886    let deadline = Instant::now() + max_wait;
11887    loop {
11888        if pids.iter().all(|&pid| !alive(pid)) {
11889            return;
11890        }
11891        if Instant::now() >= deadline {
11892            return;
11893        }
11894        tokio::time::sleep(poll).await;
11895    }
11896}
11897
11898/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11899/// admission that it could not even get a verify command to run, as opposed
11900/// to evidence the command actually produced? A caller that would otherwise
11901/// read a resource-blocked outcome as a red command must check this first:
11902/// see [`Runner::gate`], which retries rather than records `Blocked` when
11903/// this is true.
11904fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11905    outcomes.iter().any(|o| o.resource_blocked)
11906}
11907
11908/// What [`Runner::gate_fix_round`] decided.
11909pub(crate) enum GateFix {
11910    /// The tree changed and `verify.e2e` is still green: run the gate again.
11911    Retry,
11912    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11913    /// last failure stands and the run ends blocked.
11914    Stop,
11915    /// `verify.e2e` could not run after the fix (magi's own contention):
11916    /// decide nothing now, a later reentry retries.
11917    Defer,
11918}
11919
11920/// Is every red command in `outcomes` an ordinary failure the code could
11921/// explain: it ran, exited non-zero, and said something?
11922///
11923/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11924/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11925/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11926/// what the command is stays the gate's business.
11927pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11928    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11929    red.peek().is_some()
11930        && red.all(|o| {
11931            !o.resource_blocked
11932                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11933                && !o.output_tail.trim().is_empty()
11934        })
11935}
11936
11937/// Describe one verify command's outcome for the event log, distinguishing a
11938/// build/link failure — the toolchain never produced a binary to run — from
11939/// an actual test failure, since only the latter is a verdict on the patch.
11940fn e2e_outcome_label(o: &CommandOutcome) -> String {
11941    if o.ok() {
11942        return "pass".to_owned();
11943    }
11944    let reason = if o.build_failed() {
11945        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11946    } else {
11947        format!("FAIL ({:?})", o.code)
11948    };
11949    format!(
11950        "{reason}\n{}",
11951        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11952    )
11953}
11954
11955/// Run `verify.e2e`, retrying once if the first attempt could not build or
11956/// link — a build/link failure is frequently a race against a shared
11957/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11958/// `verify` event per command, tagged with `context` (normally `"round N"`)
11959/// so the two call sites that need this — the ordinary per-round leg in
11960/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11961/// it will ever call a round green — read identically in the event log.
11962async fn run_e2e_with_retry(
11963    state: &mut RunState,
11964    shell: &[String],
11965    commands: &[String],
11966    worktree: &Path,
11967    timeout: Duration,
11968    context: &str,
11969) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11970    let (mut e2e, mut timed_out_pids) = run_commands(
11971        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11972    )
11973    .await;
11974    for o in &e2e {
11975        state.event(
11976            "verify",
11977            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11978        );
11979    }
11980    // A build/link failure is not a verdict on the patch — it is frequently a
11981    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11982    // one retry before letting a red like that decide the round.
11983    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11984    if verify_retried {
11985        state.event(
11986            "verify",
11987            format!(
11988                "{context}: verify could not build/link, not a test result — retrying once \
11989                 before concluding"
11990            ),
11991        );
11992        let retried = run_commands(
11993            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11994        )
11995        .await;
11996        e2e = retried.0;
11997        // Both attempts' timeouts matter, not just the last one: the first
11998        // attempt's descendants may still be alive alongside the retry's.
11999        timed_out_pids.extend(retried.1);
12000        for o in &e2e {
12001            state.event(
12002                "verify",
12003                format!(
12004                    "{context}: retry `{}` -> {}",
12005                    o.command,
12006                    e2e_outcome_label(o)
12007                ),
12008            );
12009        }
12010    }
12011    (e2e, verify_retried, timed_out_pids)
12012}
12013
12014/// Run configured shell commands in `cwd`, in order. The second element is
12015/// the pid of every command that hit `timeout` and was still running when
12016/// this stopped waiting on it (best-effort: `None` when the platform did not
12017/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12018/// that releases a shared resource afterward needs to know.
12019///
12020/// Records `task` into [`RunState::active`] at every command boundary
12021/// (`RunState::task_command`) and clears it once the whole list has run
12022/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12023/// for minutes with no seat and no output of its own to show for it (see
12024/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12025/// apart from "not yet run" without this), and this is the only place that
12026/// knows which command is running right now and how many are left. Three
12027/// saves per command — start, not per second — matching the same "only at a
12028/// boundary" rule [`wave`] already follows for seats.
12029#[allow(clippy::too_many_arguments)]
12030async fn run_commands(
12031    state: &mut RunState,
12032    node: &str,
12033    task: &str,
12034    attempt: usize,
12035    shell: &[String],
12036    commands: &[String],
12037    cwd: &Path,
12038    timeout: Duration,
12039) -> (Vec<CommandOutcome>, Vec<u32>) {
12040    if commands.is_empty() {
12041        // Nothing to mark as running and nothing to clear — an empty list
12042        // means "not configured", and touching `active` (or the disk) over
12043        // that would be a write for every round of a repo with no
12044        // `verify.e2e` / `verify.gate` commands at all.
12045        return (Vec::new(), Vec::new());
12046    }
12047    let mut out = Vec::new();
12048    let mut timed_out_pids = Vec::new();
12049    let total = commands.len();
12050    for (idx, command) in commands.iter().enumerate() {
12051        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12052        if let Err(e) = state.save() {
12053            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12054        }
12055        let started = Instant::now();
12056        let mut cmd = tokio::process::Command::new(&shell[0]);
12057        cmd.quiet();
12058        // Verify the checkout as a standalone caller. Worker ownership and
12059        // pinned executable selection belong to this driver, not to tests or
12060        // tools it runs. Preserve home/seat context and Running's containment.
12061        for key in crate::worker::INVOCATION_ENV {
12062            cmd.env_remove(key);
12063        }
12064        cmd.args(&shell[1..])
12065            .arg(command)
12066            .current_dir(cwd)
12067            .stdin(std::process::Stdio::null())
12068            .stdout(std::process::Stdio::piped())
12069            .stderr(std::process::Stdio::piped())
12070            .kill_on_drop(true);
12071        let mut outputs = Vec::new();
12072        let protected = (|| -> Result<()> {
12073            crate::incremental::configure(&mut cmd, cwd)?;
12074            let mut paths = vec![cwd.to_owned()];
12075            if let Some(cache) = state.config.cache_dir() {
12076                paths.push(cache);
12077            }
12078            outputs.extend(crate::retention::OutputLease::admit(
12079                &crate::run::home(),
12080                &paths,
12081                state.config.disk.min_free_bytes,
12082            )?);
12083            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12084                outputs.push(guard);
12085            }
12086            if let Some(cache) = state.config.cache_dir() {
12087                outputs.push(crate::retention::OutputLease::cache(
12088                    &crate::run::home(),
12089                    &cache,
12090                    false,
12091                )?);
12092            }
12093            Ok(())
12094        })();
12095        if let Err(e) = protected {
12096            let detail = format!("execution admission unavailable: {e:#}");
12097            state.resource_block = Some(crate::run::ResourceBlock {
12098                cause: crate::run::ResourceBlockCause::Other,
12099                node: node.to_owned(),
12100                kind: if detail.contains("disk admission") {
12101                    "disk admission"
12102                } else {
12103                    "output ownership"
12104                }
12105                .to_owned(),
12106                detail: detail.clone(),
12107                at: Timestamp::now(),
12108            });
12109            out.push(CommandOutcome {
12110                command: command.clone(),
12111                code: None,
12112                output_tail: detail,
12113                duration_ms: 0,
12114                resource_blocked: true,
12115            });
12116            break;
12117        }
12118        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12119        let (code, body) = match spawned {
12120            Ok(mut running) => {
12121                let pid = running.child().id();
12122                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12123                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12124                let mut paths = vec![cwd.to_owned()];
12125                if let Some(cache) = state.config.cache_dir() {
12126                    paths.push(cache);
12127                }
12128                let mut pressure = None;
12129                let status = tokio::select! {
12130                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12131                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12132                        pressure = Some(reason); None
12133                    }
12134                };
12135                let (code, failure) = match status {
12136                    Some(Ok(Ok(status))) => (
12137                        status.code(),
12138                        status
12139                            .code()
12140                            .is_none()
12141                            .then(|| format!("command terminated: {status}")),
12142                    ),
12143                    Some(Ok(Err(error))) => {
12144                        (None, Some(format!("failed waiting for command: {error}")))
12145                    }
12146                    Some(Err(_)) => {
12147                        // Leased cleanup confirms process-tree death itself.
12148                        // Unleased commands still need the caller's PID check
12149                        // after kill-on-drop, before releasing cache ownership.
12150                        if !running.coordinated()
12151                            && let Some(pid) = pid
12152                        {
12153                            timed_out_pids.push(pid);
12154                        }
12155                        (
12156                            None,
12157                            Some(format!("timed out after {}s", timeout.as_secs())),
12158                        )
12159                    }
12160                    None => (None, None),
12161                };
12162                running.finish().await;
12163                if let Some(reason) = pressure {
12164                    state.resource_block = Some(crate::run::ResourceBlock {
12165                        cause: crate::run::ResourceBlockCause::Other,
12166                        node: node.to_owned(),
12167                        kind: "disk pressure".to_owned(),
12168                        detail: reason.clone(),
12169                        at: Timestamp::now(),
12170                    });
12171                    out.push(CommandOutcome {
12172                        command: command.clone(),
12173                        code: None,
12174                        output_tail: reason,
12175                        duration_ms: started.elapsed().as_millis() as u64,
12176                        resource_blocked: true,
12177                    });
12178                    break;
12179                }
12180                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12181                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12182                if let Some(failure) = failure {
12183                    body.push('\n');
12184                    body.push_str(&failure);
12185                }
12186                (code, body)
12187            }
12188            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12189        };
12190        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12191        if !body.is_empty() {
12192            // The full text is kept whatever the summary dropped. A fresh
12193            // name per command so a resume or retry never overwrites an
12194            // earlier run's output. A failed write changes no verdict.
12195            match crate::run::write_verification_artifact(state, task, &body) {
12196                Ok(name) if summary != body => {
12197                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12198                }
12199                Ok(_) => {}
12200                Err(e) => {
12201                    let warning = format!("could not save the full {task} output: {e:#}");
12202                    tracing::warn!("{warning}");
12203                    state.event(node, warning);
12204                    if summary != body {
12205                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12206                    }
12207                }
12208            }
12209        }
12210        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12211        out.push(CommandOutcome {
12212            command: command.clone(),
12213            code,
12214            output_tail: summary,
12215            duration_ms: started.elapsed().as_millis() as u64,
12216            resource_blocked,
12217        });
12218        if resource_blocked {
12219            state.resource_block = Some(crate::run::ResourceBlock {
12220                cause: crate::run::ResourceBlockCause::Other,
12221                node: node.to_owned(),
12222                kind: "incremental cache admission".into(),
12223                detail: body,
12224                at: Timestamp::now(),
12225            });
12226            break;
12227        }
12228    }
12229    state.task_finished(task);
12230    if let Err(e) = state.save() {
12231        tracing::warn!("could not persist the end of {task}: {e:#}");
12232    }
12233    (out, timed_out_pids)
12234}
12235
12236/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12237/// section (`report::run`) and in the `merge` event this node records — for
12238/// the operator to run by hand.
12239///
12240/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12241/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12242/// merge commits", or "require linear history") rejects the push a `--no-ff`
12243/// merge would produce, which is exactly the guidance this function replaces.
12244/// `message`'s first line becomes the squash commit's subject, matching the
12245/// note `report::run` prints alongside this command — see that function for
12246/// why an explicit subject is not optional there.
12247fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12248    let repo = repo.display();
12249    match style {
12250        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12251        MergeStyle::Squash => {
12252            // The subject sits inside double quotes, and a title an agent
12253            // wrote may carry the characters that break out of them.
12254            let subject = message
12255                .lines()
12256                .next()
12257                .unwrap_or(branch)
12258                .replace(['\\', '"', '$', '`'], "");
12259            format!(
12260                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12261            )
12262        }
12263        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12264    }
12265}
12266
12267/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12268/// under the hood, rejects a `title` over 256 characters and the whole
12269/// command fails — no PR at all, for a run whose body was otherwise fine
12270/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12271/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12272/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12273/// this bound. It is a margin, not a guarantee — a title packed
12274/// with multi-unit characters could still in principle land close to the
12275/// edge, but a real task title's occasional emoji or accented letter fits
12276/// comfortably inside it.
12277const PR_TITLE_MAX: usize = 240;
12278
12279/// A pull request title from the opening line of `text`, or `None` when that
12280/// line is not English (GitHub text is) or has no letters.
12281///
12282/// A line within `max` is kept as is. A longer one is cut at the end of its
12283/// first sentence when that falls inside `max`, else at a word boundary with a
12284/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12285/// make every merely-truncated title look non-English). The language check
12286/// runs on the kept text before any mark is added, so only what GitHub will
12287/// show is judged: an English opening followed by non-ASCII far past the cut
12288/// still passes.
12289fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12290    let line = queue::first_line(text)?;
12291    let chars: Vec<char> = line.chars().collect();
12292    let (kept, mark) = if chars.len() <= max {
12293        (line.to_owned(), "")
12294    } else if let Some(end) = sentence_end(&chars, max) {
12295        (chars[..end].iter().collect::<String>(), "")
12296    } else {
12297        let room = max.saturating_sub(3);
12298        // Cut at the last space inside the room; when the char just past the
12299        // room is a space the room already ends on a word.
12300        let cut = if chars[room].is_whitespace() {
12301            room
12302        } else {
12303            chars[..room]
12304                .iter()
12305                .rposition(|c| c.is_whitespace())
12306                .unwrap_or(room)
12307        };
12308        let head: String = chars[..cut].iter().collect();
12309        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12310        (head.to_owned(), "...")
12311    };
12312    if kept.is_empty() {
12313        return None;
12314    }
12315    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12316        return None;
12317    }
12318    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12319        return None;
12320    }
12321    Some(format!("{kept}{mark}"))
12322}
12323
12324/// The char length of the first sentence of `chars` when it ends within `max`
12325/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12326/// abbreviations so `e.g. foo` does not end a title early.
12327fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12328    const MIN: usize = 20;
12329    for i in MIN..max.min(chars.len()) {
12330        if !matches!(chars[i], '.' | '!' | '?') {
12331            continue;
12332        }
12333        let Some(&next) = chars.get(i + 1) else {
12334            continue;
12335        };
12336        if !next.is_whitespace() {
12337            continue;
12338        }
12339        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12340        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12341            continue;
12342        }
12343        let word: String = chars[..i]
12344            .iter()
12345            .rev()
12346            .take_while(|c| !c.is_whitespace())
12347            .collect::<Vec<_>>()
12348            .into_iter()
12349            .rev()
12350            .collect();
12351        let word = word.to_ascii_lowercase();
12352        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12353            continue;
12354        }
12355        let end = chars[..i]
12356            .iter()
12357            .rposition(|c| !c.is_whitespace())
12358            .map_or(i, |p| p + 1);
12359        return Some(end);
12360    }
12361    None
12362}
12363
12364/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12365/// change: a title and a body describing what was *implemented*, not the task
12366/// that asked for it. A task reads as a request; a reader of the merged
12367/// history wants the change.
12368struct PrMessage {
12369    title: String,
12370    body: String,
12371}
12372
12373impl PrMessage {
12374    /// Title, blank line, body. The first line is the squash/merge commit
12375    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12376    /// has to stay one sensible line.
12377    fn commit_message(&self) -> String {
12378        format!("{}\n\n{}", self.title, self.body)
12379    }
12380}
12381
12382/// The text after a leading `TITLE:` (any case) on `line`.
12383fn title_marker(line: &str) -> Option<&str> {
12384    let line = line.trim();
12385    let head = line.get(..6)?;
12386    head.eq_ignore_ascii_case("title:")
12387        .then(|| line[6..].trim())
12388}
12389
12390/// The implementer's own one-line title: the `TITLE:` line the implement
12391/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12392/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12393/// source, and a title that says as much is refused here too.
12394fn summary_title(summary: &str) -> Option<String> {
12395    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12396    let raw = title_marker(first)?;
12397    if raw.is_empty() {
12398        return None;
12399    }
12400    let title = queue::title_from(raw, PR_TITLE_MAX);
12401    let lower = title.to_ascii_lowercase();
12402    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12403        return None;
12404    }
12405    Some(title)
12406}
12407
12408/// How `open_review`'s instruction begins; see [`landing_title`].
12409const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12410
12411/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12412pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12413
12414/// Does this run review work that already existed, rather than implement a
12415/// task? Runs recorded before `reviewed_commits` existed carry only the
12416/// candidate marker.
12417pub(crate) fn is_review_run(state: &RunState) -> bool {
12418    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12419}
12420
12421/// The title of a review-only run: the subject of the oldest commit under
12422/// review. Later commits are usually fixups, and `instruction` is the review
12423/// prompt, which says nothing about the change. GitHub text is English, so a
12424/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12425fn review_title(state: &RunState) -> Option<String> {
12426    english_subject(
12427        state.reviewed_commits.as_ref()?.first()?,
12428        state.config.graph.github_english(),
12429    )
12430}
12431
12432/// `raw` as a pull request title, or `None` when it is blank, not English
12433/// (GitHub text is), or one of magi's own candidate commit subjects.
12434fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12435    let raw = raw.trim();
12436    if raw.is_empty() {
12437        return None;
12438    }
12439    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12440        return None;
12441    }
12442    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12443        return None;
12444    }
12445    let title = queue::title_from(raw, PR_TITLE_MAX);
12446    let lower = title.to_ascii_lowercase();
12447    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12448        return None;
12449    }
12450    Some(title)
12451}
12452
12453/// What a review-only run's branch says about itself, read at the moment the
12454/// pull request is opened.
12455#[derive(Debug, Clone, PartialEq, Eq)]
12456struct BranchFacts {
12457    /// `(subject, body)` of each commit, oldest first.
12458    commits: Vec<(String, String)>,
12459    /// Trimmed `git diff --stat`.
12460    stat: String,
12461}
12462
12463/// Longest diff stat shown: this many file lines plus the summary line.
12464const STAT_FILE_LINES: usize = 25;
12465/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12466const COMMITS_MAX_CHARS: usize = 20_000;
12467
12468/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12469/// say or finds nothing, so the caller falls back to what the run recorded.
12470async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12471    let commits = git::commit_log(repo, base, branch).await.ok()?;
12472    if commits.is_empty() {
12473        return None;
12474    }
12475    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12476    let lines: Vec<&str> = stat.lines().collect();
12477    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12478        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12479        let more = format!(" ... {omitted} more file(s)");
12480        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12481        kept.push(&more);
12482        kept.push(lines[lines.len() - 1]);
12483        kept.join("\n")
12484    } else {
12485        lines.join("\n")
12486    };
12487    Some(BranchFacts { commits, stat })
12488}
12489
12490/// Defang what would break the surrounding markdown: a closing `</details>`
12491/// and a code fence.
12492fn markdown_safe(text: &str) -> String {
12493    text.replace("</details>", "&lt;/details&gt;")
12494        .replace("\x60\x60\x60", "~~~")
12495}
12496
12497fn neutral_title(state: &RunState, winner: char) -> String {
12498    format!(
12499        "chore: land candidate {} of run {}",
12500        winner.to_ascii_uppercase(),
12501        state.id
12502    )
12503}
12504
12505/// Reuse the graph's bounded fixer without starting another graph or driver.
12506/// Copy the working state back even on errors so spent rounds remain visible.
12507pub(crate) async fn repair_landing_gate(
12508    state: &mut RunState,
12509    winner: &Candidate,
12510    outcomes: &[CommandOutcome],
12511) -> Result<GateFix> {
12512    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12513        return Ok(GateFix::Stop);
12514    }
12515    let mut runner = Runner {
12516        roles: state.config.resolve_roles()?,
12517        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12518        state: state.clone(),
12519        pause: Pause::new(),
12520        interrupt: Pause::new(),
12521    };
12522    let result = runner.gate_fix_round(winner, outcomes).await;
12523    *state = runner.state;
12524    result
12525}
12526
12527/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12528/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12529pub(crate) async fn verify_current_head(
12530    state: &mut RunState,
12531    winner: &Candidate,
12532    wait: &crate::repo_lock::WaitPolicy,
12533) -> Result<bool> {
12534    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12535    // Preserve the previous record before replacing it, including on crash recovery.
12536    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12537        state
12538            .verification_history
12539            .push(crate::run::VerificationEvidence {
12540                superseded_by: Some(head.clone()),
12541                archived_at: Some(Timestamp::now()),
12542                fixes_spent: Some(state.gate_fixes.len()),
12543                gate_head: None,
12544                gate_ran: false,
12545                gate: Vec::new(),
12546                e2e: state.head_e2e.take(),
12547            });
12548    }
12549    if let Some(e) = &state.head_e2e
12550        && e.head == head
12551    {
12552        if e.passed {
12553            return Ok(true);
12554        }
12555        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12556            state.status = RunStatus::Blocked;
12557            state.save()?;
12558            return Ok(false);
12559        }
12560    } else if state.reviews.last().is_some_and(|r| {
12561        r.verified_head.as_deref() == Some(head.as_str())
12562            && (r.e2e_status() == E2eStatus::Passed
12563                || (r.e2e_status() == E2eStatus::NotConfigured
12564                    && state.config.verify.e2e.is_empty()))
12565    }) || state
12566        .gate_repair_e2e
12567        .as_ref()
12568        .is_some_and(|e| e.head == head && e.passed)
12569    {
12570        return Ok(true);
12571    }
12572    if let Some(e) = &state.gate_repair_e2e
12573        && e.head == head
12574        && !e.passed
12575        && !e.outcomes.is_empty()
12576        && !verify_inconclusive(&e.outcomes)
12577    {
12578        state.status = RunStatus::Blocked;
12579        state.save()?;
12580        return Ok(false);
12581    }
12582    state.head_e2e = Some(crate::run::GateRepairE2e {
12583        head: head.clone(),
12584        passed: false,
12585        outcomes: Vec::new(),
12586    });
12587    state.save()?;
12588    let commands = state.config.verify.e2e.clone();
12589    let shell = state.config.shell();
12590    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12591    let cache_dir = state.config.cache_dir();
12592    let outcomes = if commands.is_empty() {
12593        Vec::new()
12594    } else {
12595        let (outcomes, _) = with_cache_lease(
12596            state,
12597            cache_dir.as_deref(),
12598            "e2e",
12599            "e2e",
12600            &winner.worktree,
12601            &head,
12602            timeout,
12603            "current head e2e",
12604            wait,
12605            |state, budget| async move {
12606                run_e2e_with_retry(
12607                    state,
12608                    &shell,
12609                    &commands,
12610                    &winner.worktree,
12611                    budget,
12612                    "current head e2e",
12613                )
12614                .await
12615            },
12616        )
12617        .await?;
12618        outcomes
12619    };
12620    let unresolved = verify_inconclusive(&outcomes);
12621    let passed = !unresolved
12622        && outcomes.iter().all(CommandOutcome::ok)
12623        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12624    if !outcomes.is_empty() && !unresolved {
12625        state.record_base_progress(&head);
12626    }
12627    state.head_e2e = Some(crate::run::GateRepairE2e {
12628        head: head.clone(),
12629        passed,
12630        outcomes,
12631    });
12632    if !passed && !unresolved {
12633        state.status = RunStatus::Blocked;
12634        state.event(
12635            "e2e",
12636            format!("verify.e2e failed on head {head}; nothing published"),
12637        );
12638    }
12639    state.save()?;
12640    Ok(passed)
12641}
12642
12643/// Recheck a repair's e2e requirement across failures, resource deferrals and
12644/// process interruptions. A successful result only applies to its exact SHA.
12645pub(crate) async fn check_gate_repair_e2e(
12646    state: &mut RunState,
12647    winner: &Candidate,
12648    wait: &crate::repo_lock::WaitPolicy,
12649) -> Result<GateFix> {
12650    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12651        return Ok(GateFix::Retry);
12652    }
12653    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12654    if state
12655        .gate_repair_e2e
12656        .as_ref()
12657        .is_some_and(|r| r.head == head && r.passed)
12658    {
12659        return Ok(GateFix::Retry);
12660    }
12661    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12662        head: head.clone(),
12663        passed: false,
12664        outcomes: Vec::new(),
12665    });
12666    state.save()?;
12667    let commands = state.config.verify.e2e.clone();
12668    let outcomes = if commands.is_empty() {
12669        Vec::new()
12670    } else {
12671        let shell = state.config.shell();
12672        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12673        let cache_dir = state.config.cache_dir();
12674        let (outcomes, _) = with_cache_lease(
12675            state,
12676            cache_dir.as_deref(),
12677            "e2e",
12678            "e2e",
12679            &winner.worktree,
12680            &head,
12681            timeout,
12682            "gate repair e2e",
12683            wait,
12684            |state, budget| async move {
12685                run_e2e_with_retry(
12686                    state,
12687                    &shell,
12688                    &commands,
12689                    &winner.worktree,
12690                    budget,
12691                    "gate repair e2e",
12692                )
12693                .await
12694            },
12695        )
12696        .await?;
12697        outcomes
12698    };
12699    let deferred = verify_inconclusive(&outcomes);
12700    let passed = outcomes.iter().all(CommandOutcome::ok)
12701        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12702    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12703        head,
12704        passed,
12705        outcomes,
12706    });
12707    state.save()?;
12708    if deferred {
12709        Ok(GateFix::Defer)
12710    } else if !passed {
12711        state.event(
12712            "gate",
12713            "verify.e2e failed after gate repair; nothing published",
12714        );
12715        Ok(GateFix::Stop)
12716    } else {
12717        Ok(GateFix::Retry)
12718    }
12719}
12720
12721/// Run the same configured gate for a landing head and a fresh graph.
12722pub(crate) async fn run_landing_gate(
12723    state: &mut RunState,
12724    winner: &Candidate,
12725    wait: &crate::repo_lock::WaitPolicy,
12726) -> Result<Vec<CommandOutcome>> {
12727    let shell = state.config.shell();
12728    let gate_commands = state.config.verify.gate.clone();
12729    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12730    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12731        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12732        Vec::new()
12733    } else {
12734        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12735        let cache_dir = state.config.cache_dir();
12736        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12737        let (outcomes, _) = with_cache_lease(
12738            state,
12739            cache_dir.as_deref(),
12740            "gate",
12741            "gate",
12742            &winner.worktree,
12743            &head,
12744            timeout,
12745            "final gate",
12746            wait,
12747            |state, budget| {
12748                let shell = shell.clone();
12749                let gate_commands = gate_commands.clone();
12750                let winner = winner.clone();
12751                async move {
12752                    Runner::run_pre_gate(state, &winner).await;
12753                    // Bind command evidence after mechanical fixes, before commands start.
12754                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12755                    let (outcomes, timed_out_pids) = run_commands(
12756                        state,
12757                        "gate",
12758                        "gate",
12759                        0,
12760                        &shell,
12761                        &gate_commands,
12762                        &winner.worktree,
12763                        budget,
12764                    )
12765                    .await;
12766                    (outcomes, false, timed_out_pids)
12767                }
12768            },
12769        )
12770        .await?;
12771        outcomes
12772    };
12773    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12774    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12775        let mut outcomes = outcomes;
12776        outcomes.push(CommandOutcome {
12777            command: "gate head stability".into(), code: None,
12778            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12779            duration_ms: 0, resource_blocked: false,
12780        });
12781        return Ok(outcomes);
12782    }
12783    if outcomes.is_empty() {
12784        // Nothing configured to check — distinct from every other
12785        // silence in this run's event log, since an empty `gate` alone
12786        // no longer says whether the gate ran at all (see
12787        // `RunState::gate_ran`'s own doc).
12788        state.event("gate", "no gate commands configured; not tested");
12789    }
12790    for o in &outcomes {
12791        state.event(
12792            "gate",
12793            format!(
12794                "`{}` -> {}",
12795                o.command,
12796                if o.ok() {
12797                    "pass".to_owned()
12798                } else {
12799                    format!(
12800                        "FAIL ({:?})\n{}",
12801                        o.code,
12802                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12803                    )
12804                }
12805            ),
12806        );
12807    }
12808    Ok(outcomes)
12809}
12810
12811/// The pull request title to hand to `land::merge_subject`. A review-only run
12812/// opened by an earlier build titled its pull request with the review prompt;
12813/// that title is dropped (empty, so the fallback applies) rather than landed.
12814/// Any other title, including an operator's rename, passes through untouched,
12815/// and so does every title of a run that implements a task.
12816pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12817    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12818        ""
12819    } else {
12820        pr_title
12821    }
12822}
12823
12824/// What the squash subject falls back to when the pull request title is empty
12825/// or candidate-shaped: for a review-only run the derived title, never the
12826/// review prompt held in `instruction`.
12827pub fn landing_subject_source(state: &RunState) -> String {
12828    if is_review_run(state) {
12829        let winner = state.candidates.first().map_or('A', |c| c.label);
12830        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12831    }
12832    state.instruction.clone()
12833}
12834
12835/// `summary` without its `TITLE:` line, which the pull request title already
12836/// carries.
12837fn summary_without_title(summary: &str) -> String {
12838    let mut lines = summary.trim().lines().peekable();
12839    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12840        lines.next();
12841    }
12842    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12843}
12844
12845/// The pull request title and body for the winning candidate.
12846///
12847/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12848/// the task's own opening line via [`queue::title_from`] when there is none.
12849/// `state.instruction` can open with blank lines (`task_text` only rejects a
12850/// body that is blank *entirely*), which `title_from` skips.
12851///
12852/// Publishing uses an independent description generated from the final diff
12853/// and structured evidence. Internal summaries and reports remain local.
12854/// Non-publishing previews may display the original task and candidate summary.
12855/// The footer retains the run and candidate provenance.
12856#[cfg(test)]
12857fn pr_message(state: &RunState, winner: char) -> PrMessage {
12858    pr_message_with(state, winner, None)
12859}
12860
12861/// [`pr_message`] with what the branch of a review-only run says about itself.
12862/// `facts` is ignored for a run that implements a task.
12863#[cfg(test)]
12864fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12865    let raw = pr_message_raw(
12866        state,
12867        winner,
12868        facts,
12869        crate::pr_outcome::FinalHead::Unreported,
12870        ChangeNote::NotAsked,
12871    );
12872    let id = crate::scrub::Identity::current();
12873    PrMessage {
12874        title: crate::scrub::scrub(&raw.title, &id),
12875        body: crate::scrub::scrub(&raw.body, &id),
12876    }
12877}
12878
12879/// What the public description may say about the change itself.
12880#[derive(Debug, Clone, Copy)]
12881enum ChangeNote<'a> {
12882    /// A local preview or non-publishing merge mode: no public generation needed.
12883    NotAsked,
12884    /// A summary written from the final diff.
12885    Summary(&'a crate::run::ChangeSummary),
12886    /// A summary was asked for and could not be produced; the text says why.
12887    Unavailable(&'a str),
12888}
12889
12890fn pr_message_raw(
12891    state: &RunState,
12892    winner: char,
12893    facts: Option<&BranchFacts>,
12894    head: crate::pr_outcome::FinalHead<'_>,
12895    change: ChangeNote<'_>,
12896) -> PrMessage {
12897    if !matches!(change, ChangeNote::NotAsked) {
12898        let (title, mut body) = match change {
12899            ChangeNote::Summary(c) => (
12900                c.title.clone(),
12901                format!("## Summary\n\n{}\n\n", c.body.trim()),
12902            ),
12903            ChangeNote::Unavailable(why) => (
12904                neutral_title(state, winner),
12905                format!(
12906                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12907                ),
12908            ),
12909            ChangeNote::NotAsked => unreachable!(),
12910        };
12911        body.push_str(&public_pr_evidence(state, head));
12912        body.push_str(&format!(
12913            "\n---\nmagi:run/{} magi:candidate-{}\n",
12914            state.id,
12915            winner.to_ascii_lowercase()
12916        ));
12917        return PrMessage { title, body };
12918    }
12919    pr_preview(state, winner, facts, head)
12920}
12921
12922/// Append public facts from the run records after every prose generation,
12923/// including posting-gate rewrites. The agent cannot drop this provenance.
12924fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12925    let mut body = crate::pr_outcome::review_rounds(state);
12926    body.push_str(&crate::pr_outcome::final_verification(state, head));
12927    let open = state.last_round_findings();
12928    if !open.is_empty() {
12929        body.push_str("## Open review findings\n\n");
12930        for f in open {
12931            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12932        }
12933        body.push('\n');
12934    }
12935    if let Some(c) = state.change_summary.as_ref()
12936        && head == crate::pr_outcome::FinalHead::At(&c.head)
12937    {
12938        body.push_str(&format!(
12939            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12940            c.base, c.head
12941        ));
12942    }
12943    body
12944}
12945
12946/// Local-only preview of the operator's task and candidate summary.
12947fn pr_preview(
12948    state: &RunState,
12949    winner: char,
12950    facts: Option<&BranchFacts>,
12951    head: crate::pr_outcome::FinalHead<'_>,
12952) -> PrMessage {
12953    let summary = state
12954        .candidates
12955        .iter()
12956        .find(|c| c.label == winner)
12957        .map(|c| c.summary.as_str())
12958        .unwrap_or_default();
12959    // The fallback is the operator's own words and may not be English; GitHub
12960    // text always is, so a non-English task gets a neutral title instead.
12961    let review = is_review_run(state);
12962    let title = if review {
12963        facts
12964            .and_then(|f| {
12965                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12966            })
12967            .or_else(|| review_title(state))
12968            .or_else(|| {
12969                state
12970                    .candidates
12971                    .iter()
12972                    .find(|c| c.label == winner)
12973                    .filter(|c| !c.branch.starts_with("magi/"))
12974                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12975            })
12976            .unwrap_or_else(|| neutral_title(state, winner))
12977    } else {
12978        summary_title(summary).unwrap_or_else(|| {
12979            english_title(
12980                &state.instruction,
12981                PR_TITLE_MAX,
12982                state.config.graph.github_english(),
12983            )
12984            .unwrap_or_else(|| neutral_title(state, winner))
12985        })
12986    };
12987
12988    let mut body = String::new();
12989    let what = summary_without_title(summary);
12990    if !what.is_empty() {
12991        body.push_str("## Summary\n\n");
12992        body.push_str(&what);
12993        body.push_str("\n\n");
12994    }
12995
12996    // Review history uses structured facts across all rounds, never reports.
12997    body.push_str(&crate::pr_outcome::review_rounds(state));
12998    body.push_str(&crate::pr_outcome::final_verification(state, head));
12999    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
13000
13001    let open = state.open_findings();
13002    if !open.is_empty() {
13003        body.push_str("## Open review findings\n\n");
13004        for f in &open {
13005            body.push_str(&format!(
13006                "- `{}` [{:?}] {}\n",
13007                f.id, f.severity, "Context retained in local review records."
13008            ));
13009        }
13010        body.push('\n');
13011    }
13012
13013    if let Some(fix) = fix
13014        && !fix.rejected.is_empty()
13015    {
13016        body.push_str("## Declined by the fixer\n\n");
13017        for r in &fix.rejected {
13018            body.push_str(&format!(
13019                "- `{}`: {}\n",
13020                r.id, "Reason retained in local review records."
13021            ));
13022        }
13023        body.push('\n');
13024    }
13025
13026    if review {
13027        // The review prompt is not the task; list what the branch carries.
13028        body.push_str("## Commits under review\n\n");
13029        if let Some(facts) = facts {
13030            let mut left = COMMITS_MAX_CHARS;
13031            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13032                let mut entry = format!("- {}\n", markdown_safe(subject));
13033                for l in markdown_safe(text).lines() {
13034                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13035                }
13036                if left == 0 {
13037                    body.push_str(&format!(
13038                        "- ... {} more commit(s)\n",
13039                        facts.commits.len() - i
13040                    ));
13041                    break;
13042                }
13043                if entry.len() > left {
13044                    // Even the first commit is cut: one huge body must not
13045                    // push the whole description past GitHub's limit.
13046                    let mut end = left;
13047                    while !entry.is_char_boundary(end) {
13048                        end -= 1;
13049                    }
13050                    entry.truncate(end);
13051                    entry.push_str("\n  ... (truncated)\n");
13052                    left = 0;
13053                } else {
13054                    left -= entry.len();
13055                }
13056                body.push_str(&entry);
13057            }
13058            if !facts.stat.trim().is_empty() {
13059                body.push_str(&format!(
13060                    "\n## Diff stat\n\n```\n{}\n```\n",
13061                    markdown_safe(facts.stat.trim())
13062                ));
13063            }
13064        } else {
13065            match &state.reviewed_commits {
13066                Some(subjects) => {
13067                    for s in subjects {
13068                        body.push_str(&format!("- {}\n", s.trim()));
13069                    }
13070                }
13071                None => {
13072                    // An older run kept only the prompt, with the commit list
13073                    // after its first paragraph.
13074                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13075                    body.push_str(rest.trim());
13076                    body.push('\n');
13077                }
13078            }
13079        }
13080    } else {
13081        let task = state.instruction.trim();
13082        let task = if task.is_empty() {
13083            "(empty task)"
13084        } else {
13085            task
13086        };
13087        body.push_str(&format!(
13088            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13089            task.replace("</details>", "&lt;/details&gt;")
13090        ));
13091    }
13092
13093    body.push_str(&format!(
13094        "\n---\nmagi:run/{} magi:candidate-{}\n",
13095        state.id,
13096        winner.to_ascii_lowercase()
13097    ));
13098
13099    PrMessage { title, body }
13100}
13101
13102/// Source imports have already been prepared from structured inputs. Legacy
13103/// inferred references must never instruct a seat to reconstruct those imports.
13104fn seeded_instruction(state: &RunState) -> String {
13105    let mut inputs = Vec::new();
13106    if let Some(start) = &state.sources.start_from {
13107        inputs.push(format!(
13108            "Start from `{}` at `{}` (including ancestry).",
13109            start.token, start.sha
13110        ));
13111    }
13112    for pick in &state.sources.pick_commits {
13113        inputs.push(format!(
13114            "Pick `{}` at `{}` (single commit).",
13115            pick.token, pick.sha
13116        ));
13117    }
13118    if inputs.is_empty() {
13119        state.instruction.clone()
13120    } else {
13121        format!(
13122            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13123            state.instruction,
13124            inputs.join("\n")
13125        )
13126    }
13127}
13128
13129/// Does the winner have no commits ahead of the base it would land on?
13130/// Any failure to find out reads as "not empty": the merge then behaves as it
13131/// always did rather than refusing on a guess.
13132async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13133    let base = &state.base_branch;
13134    let mut against = base.clone();
13135    if mode == MergeMode::Pr {
13136        // A pull request lands on the remote's base, never the local branch
13137        // of the same name: if that cannot be read, "not empty" is the safe
13138        // answer.
13139        let remote = &state.config.merge.remote;
13140        let tracking = format!("{remote}/{base}");
13141        let fetched = git::fetch(repo, remote, base).await;
13142        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13143            against = tracking;
13144        } else {
13145            return false;
13146        }
13147    }
13148    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13149}
13150
13151/// Why nothing was opened for an empty winner, with what the task's own
13152/// references resolved to.
13153fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13154    let mut detail = format!(
13155        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13156         nothing to open a pull request for"
13157    );
13158    detail.push_str(
13159        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13160    );
13161    detail
13162}
13163
13164/// What the `Pr` merge does once it knows whether the branch already has an
13165/// open pull request.
13166#[derive(Debug, PartialEq, Eq)]
13167enum PrPlan {
13168    Create,
13169    Adopt { url: String, title: String },
13170    Stop(String),
13171}
13172
13173/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13174/// a failed lookup -> stop with the real reason. Never guesses.
13175fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13176    match found {
13177        Ok(land::OpenPr::None) => PrPlan::Create,
13178        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13179        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13180            "several open pull requests exist for this branch, not picking one: {}",
13181            urls.join(" ")
13182        )),
13183        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13184    }
13185}
13186
13187/// `gh pr create`, returning the PR url.
13188pub(crate) async fn gh_pr_create(
13189    cwd: &Path,
13190    base: &str,
13191    head: &str,
13192    title: &str,
13193    body: &str,
13194) -> Result<String> {
13195    let out = tokio::process::Command::new("gh")
13196        .args([
13197            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13198        ])
13199        .current_dir(cwd)
13200        .quiet()
13201        .stdin(std::process::Stdio::null())
13202        .output()
13203        .await
13204        .context("spawn gh")?;
13205    if out.status.success() {
13206        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13207    } else {
13208        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13209    }
13210}
13211
13212/// Never force deletion of uncommitted work, including unknown legacy trees.
13213async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13214    match std::fs::symlink_metadata(path) {
13215        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13216            // A previous cleanup may have removed the tree before a branch
13217            // deletion or state save failed. Unregister it if still present.
13218            git::worktree_remove_clean(repo, path).await?;
13219            return Ok(());
13220        }
13221        Err(e) => return Err(e.into()),
13222        Ok(_) => {}
13223    }
13224    if !git::is_clean(path).await? {
13225        bail!(
13226            "worktree has uncommitted work; refusing removal: {}",
13227            path.display()
13228        );
13229    }
13230    if !git::worktree_remove_clean(repo, path).await? {
13231        bail!("worktree removal refused: {}", path.display());
13232    }
13233    Ok(())
13234}
13235
13236/// Tear a run's worktrees and branches down.
13237///
13238/// `home` is where the updated `run.json` is saved (via
13239/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13240/// a housekeeping pass already has its own honest `home` handed to it, and
13241/// falling through to the global here would write back through whichever
13242/// directory some other process or test pinned into that `OnceLock` first,
13243/// not the one the caller actually resolved its `runs` and `state` from.
13244pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13245    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13246        bail!("refusing fold: {reason}");
13247    }
13248    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13249        bail!("candidate preservation incomplete; refusing cleanup");
13250    }
13251    // Serialize ordinary folding with the same starts/resumes as target-only
13252    // eviction. Otherwise a second janitor could remove the whole worktree
13253    // around a live output borrower even though target eviction was fenced.
13254    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13255    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13256    let _driver = crate::driver::maintenance(home, &state.id)?;
13257    let mut output_guards = Vec::new();
13258    for entry in std::fs::read_dir(state.worktree_root())
13259        .into_iter()
13260        .flatten()
13261        .flatten()
13262    {
13263        if entry.path().join(".git").is_file() {
13264            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13265        }
13266    }
13267    let repo = state.repo.clone();
13268    // CLI, Web and automatic cleanup all enter here, independently of
13269    // Runner::fold_losers. Protect deletion and shared config release too.
13270    let wait = repository_wait(state, None);
13271    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13272        Ok(lease) => lease,
13273        Err(e) => {
13274            state.save_under(home)?;
13275            return Err(e);
13276        }
13277    };
13278    // Confirmed merges dispose of their owned sources after durable evidence
13279    // capture. Preflight every source before removing anything.
13280    let _source_outputs = if state.status == RunStatus::Merged
13281        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13282    {
13283        verify_merged_fold_sources(state, drop_winner, home).await?
13284    } else {
13285        Vec::new()
13286    };
13287    let root = state.worktree_root();
13288    let winner = state.tally.as_ref().map(|t| t.winner);
13289    let mut removed = Vec::new();
13290
13291    for i in 0..state.candidates.len() {
13292        let c = state.candidates[i].clone();
13293        let is_winner = Some(c.label) == winner;
13294        if is_winner && !drop_winner {
13295            continue;
13296        }
13297        let existed = c.worktree.exists();
13298        // A vanished directory can still have a Git registration that keeps
13299        // its branch checked out. Remove that registration before the branch.
13300        remove_preserved_worktree(&repo, &c.worktree).await?;
13301        if existed {
13302            removed.push(c.worktree.to_string_lossy().into_owned());
13303        }
13304        // A branch handed to a later run (and its pull request) is not this
13305        // run's to delete.
13306        let handed_over = state.released_branches.contains(&c.branch);
13307        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13308            if !git::branch_delete(&repo, &c.branch).await? {
13309                bail!("candidate branch removal refused");
13310            }
13311            removed.push(c.branch.clone());
13312        }
13313        state.candidates[i].folded = true;
13314    }
13315
13316    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13317        let path = name.path();
13318        let keep = !drop_winner
13319            && winner.is_some_and(|w| {
13320                path.file_name()
13321                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13322            });
13323        if keep {
13324            continue;
13325        }
13326        remove_preserved_worktree(&repo, &path).await?;
13327        removed.push(path.to_string_lossy().into_owned());
13328    }
13329
13330    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13331    // judge worktrees, so once the loop above has cleared all of them out,
13332    // the parent is a bare directory nobody else was ever going to remove -
13333    // git only ever managed what was inside it. Left alone, one of these
13334    // accumulates per fully-folded run; the operator's own machine had 74.
13335    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13336    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13337    // so does anything a run never claimed that happens to share the bay.
13338    remove_if_empty(&root);
13339
13340    if state.enabled_worktree_config && drop_winner {
13341        // A release, not a raw disable: some sibling run in this repository
13342        // may still hold its own reference (see `git::acquire_worktree_config`),
13343        // and only the last release actually turns the setting back off.
13344        git::release_worktree_config(&repo).await.ok();
13345        state.enabled_worktree_config = false;
13346    }
13347    state.save_under(home)?;
13348    Ok(removed)
13349}
13350
13351async fn verify_merged_fold_sources(
13352    state: &RunState,
13353    drop_winner: bool,
13354    home: &Path,
13355) -> Result<Vec<std::fs::File>> {
13356    let winner = state.tally.as_ref().map(|t| t.winner);
13357    let mut heads = BTreeSet::new();
13358    for candidate in &state.candidates {
13359        if Some(candidate.label) == winner && !drop_winner {
13360            continue;
13361        }
13362        anyhow::ensure!(
13363            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13364            "candidate recovery source is outside the inventoried run root: {}",
13365            candidate.worktree.display()
13366        );
13367        match std::fs::symlink_metadata(&candidate.worktree) {
13368            Ok(_) => {}
13369            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13370            Err(error) => return Err(error.into()),
13371        }
13372        if !state.released_branches.contains(&candidate.branch)
13373            && git::branch_exists(&state.repo, &candidate.branch).await?
13374        {
13375            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13376        }
13377    }
13378    let mut cargo_guards = Vec::new();
13379    let entries = match std::fs::read_dir(state.worktree_root()) {
13380        Ok(entries) => Some(entries),
13381        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13382        Err(error) => return Err(error.into()),
13383    };
13384    for entry in entries.into_iter().flatten() {
13385        let path = entry?.path();
13386        let metadata = std::fs::symlink_metadata(&path)?;
13387        anyhow::ensure!(
13388            metadata.is_dir() && !metadata.file_type().is_symlink(),
13389            "non-directory or symlink recovery source retained: {}",
13390            path.display()
13391        );
13392        if !drop_winner
13393            && winner.is_some_and(|w| {
13394                path.file_name()
13395                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13396            })
13397        {
13398            continue;
13399        }
13400        anyhow::ensure!(
13401            path.join(".git").is_file(),
13402            "unregistered recovery source retained: {}",
13403            path.display()
13404        );
13405        anyhow::ensure!(
13406            git::git(
13407                &path,
13408                &[
13409                    "status",
13410                    "--porcelain",
13411                    "--untracked-files=all",
13412                    "--ignored",
13413                    "--",
13414                    ".",
13415                    ":(exclude)target"
13416                ]
13417            )
13418            .await?
13419            .is_empty(),
13420            "source changes or ignored recovery files retained: {}",
13421            path.display()
13422        );
13423        match std::fs::symlink_metadata(path.join("target")) {
13424            Ok(_) => {
13425                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13426            }
13427            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13428            Err(error) => return Err(error.into()),
13429        }
13430        for checkpoint in [
13431            "rebase-merge",
13432            "rebase-apply",
13433            "sequencer",
13434            "CHERRY_PICK_HEAD",
13435            "MERGE_HEAD",
13436            "REBASE_HEAD",
13437            "REVERT_HEAD",
13438            "BISECT_LOG",
13439            "AUTO_MERGE",
13440            "MERGE_MSG",
13441        ] {
13442            let location = git::git(
13443                &path,
13444                &[
13445                    "rev-parse",
13446                    "--path-format=absolute",
13447                    "--git-path",
13448                    checkpoint,
13449                ],
13450            )
13451            .await?;
13452            anyhow::ensure!(
13453                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13454                if e.kind() == std::io::ErrorKind::NotFound),
13455                "Git recovery checkpoint retained: {checkpoint}"
13456            );
13457        }
13458        heads.insert(git::rev_parse(&path, "HEAD").await?);
13459    }
13460    crate::fold_evidence::preserve(state, home, &heads).await?;
13461
13462    Ok(cargo_guards)
13463}
13464
13465/// Remove `dir` if it exists and has nothing in it.
13466///
13467/// Best-effort and silent by design: a directory that is not empty (a run
13468/// whose winner is still parked there, a stray file some other process left)
13469/// is exactly the case this must refuse, and a directory that is already gone
13470/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13471/// already refuses a non-empty directory, so the emptiness check below is
13472/// belt, not suspenders - it is what keeps this from ever attempting the
13473/// removal in the case that matters, rather than trusting `remove_dir`'s
13474/// error path to have no side effects if it ever changed.
13475fn remove_if_empty(dir: &Path) {
13476    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13477        std::fs::remove_dir(dir).ok();
13478    }
13479}
13480
13481/// Severity of the worst open finding in the last review round, for reporting.
13482pub fn worst_open(state: &RunState) -> Option<Severity> {
13483    state
13484        .reviews
13485        .last()?
13486        .reviews
13487        .iter()
13488        .flat_map(|r| r.findings.iter())
13489        .map(|f| f.severity)
13490        .max()
13491}
13492
13493/// `[worktree] setup` hides its products per worktree, which needs
13494/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13495/// worktrees, like the hook's; every place that runs setup calls this first.
13496async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13497    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13498        git::acquire_worktree_config(repo).await?;
13499        state.enabled_worktree_config = true;
13500    }
13501    Ok(())
13502}
13503
13504#[cfg(test)]
13505mod tests {
13506    #[tokio::test]
13507    async fn launcher_base_fetch_requires_remote_success() {
13508        let tmp = tempfile::tempdir().unwrap();
13509        let repo = tmp.path().join("repo");
13510        let remote = tmp.path().join("remote");
13511        std::fs::create_dir(&repo).unwrap();
13512        std::fs::create_dir(&remote).unwrap();
13513        git::git(&remote, &["init", "--bare", "-b", "main"])
13514            .await
13515            .unwrap();
13516        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13517        git::git(
13518            &repo,
13519            &[
13520                "-c",
13521                "user.name=Fixture",
13522                "-c",
13523                "user.email=fixture@example.invalid",
13524                "commit",
13525                "--allow-empty",
13526                "-m",
13527                "fixture",
13528            ],
13529        )
13530        .await
13531        .unwrap();
13532        git::git(
13533            &repo,
13534            &["remote", "add", "origin", remote.to_str().unwrap()],
13535        )
13536        .await
13537        .unwrap();
13538        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13539        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13540        assert_eq!(
13541            resolve_base(&repo, "main", "origin", None)
13542                .await
13543                .unwrap()
13544                .commit,
13545            head
13546        );
13547        #[cfg(unix)]
13548        {
13549            let policy: crate::config::Coordination = toml::from_str(
13550                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13551            ).unwrap();
13552            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13553            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13554            let held = crate::repo_lock::SeatLease::operation_with(
13555                &repo,
13556                Some(&policy),
13557                &crate::repo_lock::WaitPolicy::default(),
13558            )
13559            .await
13560            .unwrap();
13561            assert!(held.is_some());
13562            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13563            tokio::pin!(fetch);
13564            assert!(
13565                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13566                    .await
13567                    .is_err()
13568            );
13569            assert!(
13570                !repo.join(".git/FETCH_HEAD").exists(),
13571                "no fetch before admission"
13572            );
13573            drop(held);
13574            let fetched = fetch.await.unwrap();
13575            assert_eq!(fetched.commit, head);
13576            assert_eq!(fetched.remote, "origin");
13577            assert_eq!(fetched.branch, "main");
13578        }
13579        // A surviving tracking ref is not successful fetch evidence.
13580        git::git(
13581            &repo,
13582            &[
13583                "remote",
13584                "set-url",
13585                "origin",
13586                tmp.path().join("missing").to_str().unwrap(),
13587            ],
13588        )
13589        .await
13590        .unwrap();
13591        let error = resolve_base(&repo, "main", "origin", None)
13592            .await
13593            .unwrap_err()
13594            .to_string();
13595        assert!(error.contains("cannot read origin/main"));
13596        assert!(error.contains("refusing to branch off"));
13597    }
13598
13599    #[test]
13600    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13601        let leaked = vec!["chore(deps): update a crate".to_owned()];
13602        let own = "fix(daemon): apply a chosen action";
13603        assert!(should_retitle("", own, &leaked));
13604        assert!(should_retitle(
13605            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13606            own,
13607            &leaked
13608        ));
13609        assert!(should_retitle(
13610            "chore: land candidate A of run 1",
13611            own,
13612            &leaked
13613        ));
13614        assert!(should_retitle(
13615            "magi: candidate A (uncommitted work)",
13616            own,
13617            &leaked
13618        ));
13619        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13620        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13621        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13622    }
13623
13624    #[test]
13625    fn pr_merge_plan_creates_adopts_or_stops() {
13626        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13627        assert_eq!(
13628            pr_merge_plan(Ok(land::OpenPr::One {
13629                url: "u".into(),
13630                title: "t".into()
13631            })),
13632            PrPlan::Adopt {
13633                url: "u".into(),
13634                title: "t".into()
13635            }
13636        );
13637        let PrPlan::Stop(many) =
13638            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13639        else {
13640            panic!("many must stop");
13641        };
13642        assert!(many.contains('a') && many.contains('b'));
13643        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13644            panic!("a failed lookup must stop");
13645        };
13646        assert!(err.contains("bad token"));
13647    }
13648
13649    use super::*;
13650    use crate::run::GateStatus;
13651    use std::collections::BTreeMap;
13652    use std::time::Duration;
13653
13654    fn conductor() -> AgentSpec {
13655        AgentSpec {
13656            id: "conductor".to_owned(),
13657            kind: crate::config::AgentKind::Command,
13658            model: None,
13659            command: vec!["true".to_owned()],
13660            extra_args: Vec::new(),
13661            env: BTreeMap::new(),
13662            prompt_delivery: None,
13663            max_turn_secs: None,
13664        }
13665    }
13666
13667    fn spec(id: &str) -> AgentSpec {
13668        AgentSpec {
13669            id: id.to_owned(),
13670            kind: crate::config::AgentKind::Command,
13671            model: None,
13672            command: vec!["true".to_owned()],
13673            extra_args: Vec::new(),
13674            env: BTreeMap::new(),
13675            prompt_delivery: None,
13676            max_turn_secs: None,
13677        }
13678    }
13679
13680    fn ids(xs: &[&str]) -> BTreeSet<String> {
13681        xs.iter().map(|s| (*s).to_owned()).collect()
13682    }
13683
13684    #[test]
13685    fn pick_successor_skips_an_agent_another_seat_holds() {
13686        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13687        let roster = [spec("a"), spec("b"), spec("c")];
13688        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13689        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13690    }
13691
13692    #[test]
13693    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13694        // The other seat started on c but was handed to d; c is free again.
13695        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13696        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13697        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13698        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13699        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13700    }
13701
13702    #[test]
13703    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13704        let roster = [spec("a"), spec("b")];
13705        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13706        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13707        let carried = ids(&["b"]);
13708        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13709        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13710    }
13711
13712    #[test]
13713    fn pick_successor_returns_none_without_any_untried_successor() {
13714        let roster = [spec("a"), spec("b")];
13715        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13716        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13717        let carried = ids(&["a"]);
13718        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13719    }
13720
13721    #[test]
13722    fn pick_successor_rescue_avoids_another_seats_occupant() {
13723        // b is a carried failure and free; a is held by the other seat.
13724        let roster = [spec("a"), spec("b"), spec("c")];
13725        let carried = ids(&["b", "c"]);
13726        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13727        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13728    }
13729
13730    #[test]
13731    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13732        let roster = [spec("a"), spec("b"), spec("c")];
13733        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13734        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13735    }
13736
13737    #[test]
13738    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13739        let roster = [spec("a"), spec("b"), spec("c")];
13740        let failed = ids(&["a", "b", "c"]);
13741        // Rescue looks at the whole roster, once per id, then runs out.
13742        let mut tried = ids(&["b"]);
13743        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13744        assert_eq!(first.id, "a");
13745        tried.insert(first.id.clone());
13746        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13747        assert_eq!(second.id, "c");
13748        tried.insert(second.id.clone());
13749        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13750    }
13751
13752    #[test]
13753    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13754        let roster = [spec("a"), spec("b")];
13755        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13756        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13757    }
13758
13759    #[test]
13760    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13761        let roster = [spec("a"), spec("b"), spec("c")];
13762        let h = SeatHistory {
13763            failed: ids(&["a"]),
13764            last_ok: Some("b".to_owned()),
13765            last_fail: None,
13766        };
13767        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13768        // A last answerer that left the roster, or later failed, is ignored.
13769        let gone = SeatHistory {
13770            last_ok: Some("zzz".to_owned()),
13771            ..h.clone()
13772        };
13773        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13774        let failed = SeatHistory {
13775            failed: ids(&["a", "b"]),
13776            last_ok: Some("b".to_owned()),
13777            last_fail: None,
13778        };
13779        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13780    }
13781
13782    #[test]
13783    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13784        let first = SeatState::new("review-1", "alpha", 7);
13785        let next = handover_seat("review-1", "gamma", 7);
13786        assert_ne!(first.claude_session, next.claude_session);
13787    }
13788
13789    #[test]
13790    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13791        let mut state = state_with_summary("x", "y");
13792        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13793        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13794        assert_ne!(a.claude_session, b.claude_session);
13795    }
13796
13797    #[test]
13798    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13799        let roster = [spec("a"), spec("b")];
13800        let h = SeatHistory {
13801            failed: ids(&["a", "b"]),
13802            ..SeatHistory::default()
13803        };
13804        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13805        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13806        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13807    }
13808
13809    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13810    // fallback loop depends on to terminate: it must walk forward from the
13811    // seat's own position, never restart at the front of the roster, and it
13812    // must never hand back an id already tried, however many times that id
13813    // happens to appear.
13814
13815    #[test]
13816    fn failure_signature_ignores_numbers_and_paths() {
13817        assert_eq!(
13818            failure_signature("exited with Some(2) and no usable output"),
13819            failure_signature("exited with Some(137) and no usable output")
13820        );
13821        assert_eq!(
13822            failure_signature("process exited with no usable output (exit code 2)"),
13823            failure_signature("process exited with no usable output (exit code 137)")
13824        );
13825        for (old, new) in [
13826            (
13827                "exited with Some(2) and no usable output",
13828                "process exited with no usable output (exit code 2)",
13829            ),
13830            (
13831                "exited with None and no usable output",
13832                "process exited with no usable output (exit code unavailable)",
13833            ),
13834        ] {
13835            let previous = FailClass::Other(failure_signature(old));
13836            let current = FailClass::Other(failure_signature(new));
13837            assert_eq!(previous, current, "resumption retains the failure class");
13838            assert!(!should_hand_over(Some(&previous), &current));
13839        }
13840        assert_eq!(
13841            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13842            failure_signature("cannot open /var/x.txt: denied")
13843        );
13844        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13845    }
13846
13847    #[test]
13848    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13849        let other = FailClass::Other("x".into());
13850        assert!(should_hand_over(None, &FailClass::Quota));
13851        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13852        assert!(should_hand_over(
13853            Some(&FailClass::Timeout),
13854            &FailClass::Timeout
13855        ));
13856        assert!(should_hand_over(None, &other));
13857        assert!(!should_hand_over(Some(&other), &other));
13858        assert!(should_hand_over(
13859            Some(&other),
13860            &FailClass::Other("y".into())
13861        ));
13862        // A quota or timeout in between ends the run of identical failures.
13863        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13864        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13865    }
13866
13867    #[test]
13868    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13869        let a = SeatState::new("judge-1", "alpha", 7);
13870        let b = handover_seat("judge-1", "beta", 7);
13871        assert_ne!(a.claude_session, b.claude_session);
13872        assert_eq!(b.turns, 0);
13873    }
13874
13875    #[test]
13876    fn a_timeout_is_classified_apart_from_other_failures() {
13877        assert_eq!(
13878            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13879            Some(FailClass::Timeout)
13880        );
13881        assert!(matches!(
13882            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13883            Some(FailClass::Other(_))
13884        ));
13885    }
13886
13887    #[test]
13888    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13889        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13890        let tried = BTreeSet::from(["beta".to_owned()]);
13891        // beta sits at index 1; the next candidate is gamma, never alpha —
13892        // which is very likely a different candidate slot's own agent.
13893        let next = next_untried_in_roster(&roster, 1, &tried);
13894        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13895    }
13896
13897    #[test]
13898    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13899        let roster = vec![spec("alpha"), spec("beta")];
13900        let tried = BTreeSet::from(["beta".to_owned()]);
13901        // beta is the roster's last entry: nothing follows it, and alpha —
13902        // earlier in the roster, almost certainly a different candidate
13903        // slot's own agent — must not be reached by wrapping back to it.
13904        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13905    }
13906
13907    #[test]
13908    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13909        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13910        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13911        // beta (index 1) and gamma (index 2, the only entry after it) have
13912        // both been tried; alpha (index 0) never has, but it comes before
13913        // beta's own position, so there is nothing further for this seat.
13914        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13915    }
13916
13917    #[test]
13918    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13919        let roster = vec![spec("a"), spec("a"), spec("b")];
13920        let tried = BTreeSet::from(["a".to_owned()]);
13921        let next = next_untried_in_roster(&roster, 0, &tried);
13922        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13923    }
13924
13925    #[test]
13926    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13927        let roster = vec![spec("a"), spec("b")];
13928        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13929        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13930    }
13931
13932    #[test]
13933    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13934        let dir = tempfile::tempdir().unwrap();
13935        let bay = dir.path().join("ffff");
13936
13937        // Not there yet: nothing to do, nothing to panic on.
13938        remove_if_empty(&bay);
13939        assert!(!bay.exists());
13940
13941        // Something still inside - the winner's worktree, or a stray file -
13942        // keeps the directory standing.
13943        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13944        remove_if_empty(&bay);
13945        assert!(bay.exists(), "non-empty directory must survive");
13946
13947        // Once the last entry is gone, so is the directory itself.
13948        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13949        remove_if_empty(&bay);
13950        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13951    }
13952
13953    // `round_is_clean` is the exact decision this task fixed: a round with a
13954    // seat that never answered must not read the same as a round every seat
13955    // actually reviewed. These are deterministic and process-free by design —
13956    // the equivalent end-to-end check (a real reviewer timing out under a
13957    // live graph run) is a genuine race against wall-clock contention, and a
13958    // spawn slow enough to blow even a generous budget under a loaded test
13959    // run must not turn this specific regression check flaky.
13960
13961    #[test]
13962    fn a_full_panel_that_found_nothing_is_clean() {
13963        assert!(round_is_clean(
13964            0,
13965            true,
13966            2,
13967            2,
13968            0,
13969            IncompleteReviewPolicy::Block
13970        ));
13971    }
13972
13973    #[test]
13974    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13975        assert!(!round_is_clean(
13976            0,
13977            true,
13978            1,
13979            2,
13980            0,
13981            IncompleteReviewPolicy::Block
13982        ));
13983    }
13984
13985    #[test]
13986    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13987        assert!(!round_is_clean(
13988            1,
13989            true,
13990            1,
13991            2,
13992            0,
13993            IncompleteReviewPolicy::Warn
13994        ));
13995    }
13996
13997    #[test]
13998    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13999        assert!(round_is_clean(
14000            0,
14001            true,
14002            1,
14003            2,
14004            0,
14005            IncompleteReviewPolicy::Warn
14006        ));
14007    }
14008
14009    #[test]
14010    fn a_full_panel_with_an_open_finding_is_not_clean() {
14011        assert!(!round_is_clean(
14012            1,
14013            true,
14014            2,
14015            2,
14016            0,
14017            IncompleteReviewPolicy::Block
14018        ));
14019    }
14020
14021    #[test]
14022    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14023        assert!(!round_is_clean(
14024            0,
14025            false,
14026            2,
14027            2,
14028            0,
14029            IncompleteReviewPolicy::Block
14030        ));
14031    }
14032
14033    // The stall this task closes: under the default `block` policy, a seat
14034    // missing only because it was rate limited must not force a wait for a
14035    // session limit that will not lift by the next round. `round_is_clean`
14036    // is where that quorum carve-out lives; the review loop around it never
14037    // changes what a reviewer's vote or a finding's severity means.
14038
14039    #[test]
14040    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14041        // 1 of 2 answered, and the one missing was quota'd — the exact
14042        // "review-2 rate limited (quota)" shape from the field report.
14043        assert!(round_is_clean(
14044            0,
14045            true,
14046            1,
14047            2,
14048            1,
14049            IncompleteReviewPolicy::Block
14050        ));
14051    }
14052
14053    #[test]
14054    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14055        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14056        // not a quota loss (`quota_missing` stays 0) — worth another try.
14057        assert!(!round_is_clean(
14058            0,
14059            true,
14060            1,
14061            2,
14062            0,
14063            IncompleteReviewPolicy::Block
14064        ));
14065    }
14066
14067    #[test]
14068    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14069        assert!(!round_is_clean(
14070            1,
14071            true,
14072            1,
14073            2,
14074            1,
14075            IncompleteReviewPolicy::Block
14076        ));
14077        assert!(!round_is_clean(
14078            0,
14079            false,
14080            1,
14081            2,
14082            1,
14083            IncompleteReviewPolicy::Block
14084        ));
14085    }
14086
14087    #[test]
14088    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14089        // Every seat quota'd, nobody answered: there is no panel to decide
14090        // on, so this must fall through to the existing block-and-retry
14091        // fallback rather than call an unreviewed patch clean.
14092        assert!(!round_is_clean(
14093            0,
14094            true,
14095            0,
14096            2,
14097            2,
14098            IncompleteReviewPolicy::Block
14099        ));
14100    }
14101
14102    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14103        CommandOutcome {
14104            command: "test".to_owned(),
14105            code,
14106            output_tail: String::new(),
14107            duration_ms: 0,
14108            resource_blocked,
14109        }
14110    }
14111
14112    #[cfg(unix)]
14113    #[tokio::test]
14114    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14115        ask_test_home();
14116        let temp = tempfile::tempdir().unwrap();
14117        let mut runner = runner_at(RunStatus::Gating);
14118        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14119        let commands = vec![
14120            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14121            "touch must-not-run".to_owned(),
14122        ];
14123        let (outcomes, retried, pids) = run_e2e_with_retry(
14124            &mut runner.state,
14125            &shell,
14126            &commands,
14127            temp.path(),
14128            Duration::from_secs(5),
14129            "fixture",
14130        )
14131        .await;
14132        assert_eq!(outcomes.len(), 1);
14133        assert!(outcomes[0].resource_blocked);
14134        assert!(!outcomes[0].build_failed());
14135        assert!(!retried);
14136        assert!(pids.is_empty());
14137        assert!(!temp.path().join("must-not-run").exists());
14138        assert_eq!(
14139            runner.state.resource_block.as_ref().unwrap().kind,
14140            "incremental cache admission"
14141        );
14142    }
14143
14144    #[test]
14145    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14146        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14147        assert!(
14148            !verify_inconclusive(&[outcome(Some(1), false)]),
14149            "an ordinary failure is still evidence about the patch"
14150        );
14151        assert!(verify_inconclusive(&[outcome(None, true)]));
14152        assert!(
14153            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14154            "one inconclusive outcome taints the whole batch"
14155        );
14156        assert!(!verify_inconclusive(&[]));
14157    }
14158
14159    #[tokio::test]
14160    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14161        // Alive for the first two checks, then dead - confirms the loop
14162        // actually re-polls rather than deciding once and sleeping out the
14163        // ceiling regardless.
14164        let calls = std::sync::atomic::AtomicUsize::new(0);
14165        let started = Instant::now();
14166        wait_for_pids_with(
14167            &[123],
14168            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14169            Duration::from_millis(5),
14170            Duration::from_secs(5),
14171        )
14172        .await;
14173        assert!(
14174            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14175            "must keep checking rather than deciding on the first answer"
14176        );
14177        assert!(
14178            started.elapsed() < Duration::from_secs(1),
14179            "must return the moment it is confirmed dead, not wait out the ceiling"
14180        );
14181    }
14182
14183    #[tokio::test]
14184    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14185        let started = Instant::now();
14186        wait_for_pids_with(
14187            &[123],
14188            |_| true, // never reports dead
14189            Duration::from_millis(5),
14190            Duration::from_millis(30),
14191        )
14192        .await;
14193        let elapsed = started.elapsed();
14194        assert!(
14195            elapsed >= Duration::from_millis(30),
14196            "must not give up before its own ceiling: {elapsed:?}"
14197        );
14198        assert!(
14199            elapsed < Duration::from_secs(1),
14200            "must not wait past its own ceiling either: {elapsed:?}"
14201        );
14202    }
14203
14204    #[tokio::test]
14205    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14206        let started = Instant::now();
14207        wait_for_pids_with(
14208            &[],
14209            |_| true,
14210            Duration::from_secs(5),
14211            Duration::from_secs(5),
14212        )
14213        .await;
14214        assert!(
14215            started.elapsed() < Duration::from_millis(200),
14216            "an empty pid list has nothing to confirm"
14217        );
14218    }
14219
14220    // `review_conclusion` is the exact decision the review hand-off task
14221    // fixed: a round budget spent (or a tree that stopped moving) must not
14222    // collapse into `Blocked` regardless of what verification actually
14223    // said. Deterministic and process-free for the same reason the
14224    // `round_is_clean` family above is.
14225    fn review_round(
14226        clean: bool,
14227        blocking: usize,
14228        answered: usize,
14229        expected: usize,
14230        progressed: bool,
14231        e2e_ok: bool,
14232    ) -> ReviewRound {
14233        ReviewRound {
14234            round: 1,
14235            head: "h".to_owned(),
14236            verified_head: None,
14237            verified_at: None,
14238            reviews: Vec::new(),
14239            e2e: vec![CommandOutcome {
14240                command: "test".to_owned(),
14241                code: Some(if e2e_ok { 0 } else { 1 }),
14242                output_tail: String::new(),
14243                duration_ms: 0,
14244                resource_blocked: false,
14245            }],
14246            verify_retried: false,
14247            e2e_deferred: false,
14248            e2e_defer_reason: None,
14249            fix: None,
14250            blocking,
14251            answered,
14252            expected,
14253            clean,
14254            progressed,
14255            vote_split: false,
14256            reconsideration: Vec::new(),
14257            verdict: None,
14258        }
14259    }
14260
14261    #[test]
14262    fn review_conclusion_is_none_when_nothing_has_run() {
14263        assert_eq!(review_conclusion(&[], 3), None);
14264    }
14265
14266    #[test]
14267    fn review_conclusion_is_none_while_rounds_remain() {
14268        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14269        assert_eq!(review_conclusion(&rounds, 3), None);
14270    }
14271
14272    #[test]
14273    fn review_conclusion_is_gating_once_a_round_is_clean() {
14274        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14275        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14276    }
14277
14278    #[test]
14279    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14280        let rounds = vec![
14281            review_round(false, 1, 2, 2, true, true),
14282            review_round(false, 1, 2, 2, true, true),
14283        ];
14284        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14285    }
14286
14287    #[test]
14288    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14289        let rounds = vec![
14290            review_round(false, 1, 2, 2, true, true),
14291            review_round(false, 1, 2, 2, true, false),
14292        ];
14293        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14294    }
14295
14296    #[test]
14297    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14298        // Magi never got a command to run against this round's own head — a
14299        // resource-blocked attempt, not a red one — so this must never
14300        // settle on `Blocked` the way a genuine e2e failure would. `None`
14301        // here is what tells `Runner::review_loop` to retry the check
14302        // itself rather than trust this cheap recomputation with a verdict
14303        // it cannot actually produce.
14304        let mut blocked = review_round(false, 1, 2, 2, true, false);
14305        blocked.e2e[0].resource_blocked = true;
14306        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14307        assert_eq!(review_conclusion(&rounds, 2), None);
14308    }
14309
14310    #[test]
14311    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14312        // Missing input, not a verified tree — never a hand-off candidate.
14313        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14314        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14315    }
14316
14317    #[test]
14318    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14319        let rounds = vec![
14320            review_round(false, 1, 2, 2, false, true),
14321            review_round(false, 1, 2, 2, false, true),
14322        ];
14323        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14324    }
14325
14326    fn secs(n: u64) -> Duration {
14327        Duration::from_secs(n)
14328    }
14329
14330    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14331    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14332    #[tokio::test]
14333    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14334        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14335        if std::env::var_os(CHILD).is_none() {
14336            // Other tests reconcile every run in the process-global home,
14337            // briefly taking owner locks. This ancestry test expects an idle
14338            // predecessor, so give it its own process and pinned test home.
14339            let output = std::process::Command::new(std::env::current_exe().unwrap())
14340                .args([
14341                    "--exact",
14342                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14343                    "--nocapture",
14344                ])
14345                .env(CHILD, "1")
14346                .env_remove("MAGI_WORKER_HOME")
14347                .env_remove("MAGI_WORKER_ID")
14348                .output()
14349                .unwrap();
14350            assert!(
14351                output.status.success()
14352                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14353                "{}\n{}",
14354                String::from_utf8_lossy(&output.stdout),
14355                String::from_utf8_lossy(&output.stderr)
14356            );
14357            return;
14358        }
14359        ask_test_home();
14360        let tmp = tempfile::tempdir().unwrap();
14361        let repo = tmp.path().join("repo");
14362        std::fs::create_dir_all(&repo).unwrap();
14363        init_repo(&repo);
14364        let g = |args: &[&str]| {
14365            let out = std::process::Command::new("git")
14366                .args(args)
14367                .current_dir(&repo)
14368                .quiet()
14369                .output()
14370                .unwrap();
14371            assert!(
14372                out.status.success(),
14373                "{}",
14374                String::from_utf8_lossy(&out.stderr)
14375            );
14376        };
14377        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14378        g(&["checkout", "-b", "historical"]);
14379        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14380        g(&["add", "."]);
14381        g(&["commit", "-m", "old alternative"]);
14382        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14383        g(&["checkout", "-b", "magi/task-selected", &base]);
14384        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14385        g(&["add", "."]);
14386        g(&["commit", "-m", "measurement infrastructure"]);
14387        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14388        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14389        g(&["add", "."]);
14390        g(&["commit", "-m", "dependent implementation"]);
14391        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14392        g(&["checkout", "main"]);
14393        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14394        g(&["add", "."]);
14395        g(&["commit", "-m", "unrelated base change"]);
14396        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14397        let origin = tmp.path().join("origin.git");
14398        g(&[
14399            "clone",
14400            "--bare",
14401            repo.to_str().unwrap(),
14402            origin.to_str().unwrap(),
14403        ]);
14404        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14405        g(&["fetch", "origin"]);
14406        let selection = crate::pr_recovery::Selection {
14407            url: "https://example.test/project/pull/1".into(),
14408            branch: "magi/task-selected".into(),
14409            head: head.clone(),
14410            source_head: None,
14411            source_run: None,
14412            source_proof: None,
14413            disposition: Some(crate::pr_recovery::Disposition::Review),
14414            instruction: "preserve measured behavior".into(),
14415            approved_at: jiff::Timestamp::now(),
14416        };
14417        let mut template = runner_at(RunStatus::Reviewing);
14418        template.state.repo = repo.clone();
14419        template.state.base_commit = current.clone();
14420        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14421        template.state.origin = Some(crate::run::Origin::queue("selected"));
14422        template.state.pr_head = Some(selection.branch.clone());
14423        template.state.pr_recovery = Some(selection.clone());
14424        let mut runner =
14425            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14426                .await
14427                .unwrap();
14428        let wt = runner.state.winner().unwrap().worktree.clone();
14429        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14430        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14431        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14432        assert!(runner.state.seeds.is_empty());
14433        assert_eq!(runner.state.instruction, "preserve measured behavior");
14434        runner.sync_to_base().await.unwrap();
14435        assert_eq!(
14436            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14437            "prerequisite"
14438        );
14439        assert_eq!(
14440            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14441            "measured implementation"
14442        );
14443        assert_eq!(
14444            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14445            "current main change"
14446        );
14447        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14448        assert_eq!(
14449            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14450            head
14451        );
14452
14453        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14454        let mut partial = runner_at(RunStatus::Reviewing);
14455        partial.state.repo = repo.clone();
14456        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14457        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14458            run: Some("exit 1".into()),
14459            ..Default::default()
14460        }];
14461        let selection = runner.state.pr_recovery.clone().unwrap();
14462        partial.state.pr_recovery = Some(selection.clone());
14463        partial.state.pr_head = Some(selection.branch.clone());
14464        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14465        let id = partial.state.id.clone();
14466        let error = match Runner::open_selected(
14467            &repo,
14468            partial.state,
14469            partial.roles,
14470            1,
14471            runner.state.base_commit.clone(),
14472            selection,
14473        )
14474        .await
14475        {
14476            Ok(_) => panic!("setup must fail"),
14477            Err(e) => e,
14478        };
14479        assert_eq!(
14480            error
14481                .downcast_ref::<crate::run::StartupFailure>()
14482                .unwrap()
14483                .run,
14484            id
14485        );
14486        let failed = RunState::load(&id).unwrap();
14487        assert_eq!(failed.status, RunStatus::Failed);
14488        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14489        assert!(Runner::resume(&id).is_err());
14490        runner.state.status = RunStatus::Ready;
14491        runner.state.driver_exited = true;
14492        runner.state.save().unwrap();
14493        let mut retry = runner_at(RunStatus::Reviewing);
14494        retry.state.repo = repo.clone();
14495        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14496        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14497        reselected.approved_at = jiff::Timestamp::now();
14498        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14499        retry.state.pr_head = Some(reselected.branch.clone());
14500        retry.state.pr_recovery = Some(reselected.clone());
14501        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14502            .await
14503            .unwrap();
14504        let restarted = Runner::open_selected(
14505            &repo,
14506            retry.state,
14507            retry.roles,
14508            1,
14509            runner.state.base_commit.clone(),
14510            reselected,
14511        )
14512        .await
14513        .unwrap();
14514        assert_eq!(
14515            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14516                .await
14517                .unwrap(),
14518            head
14519        );
14520        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14521        assert_eq!(
14522            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14523            head
14524        );
14525    }
14526
14527    #[tokio::test]
14528    #[cfg(unix)]
14529    async fn fallback_and_interrupted_restoration_require_coordination() {
14530        use std::os::fd::AsRawFd as _;
14531        ask_test_home();
14532        let tmp = tempfile::tempdir().unwrap();
14533        let mut runner = registered_runner(tmp.path()).await;
14534        runner.state.config.graph.timeout_resource_wait = 1;
14535        let winner = runner.state.candidates[0].clone();
14536        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14537        let leftover = winner.worktree.join("leftover");
14538        std::fs::write(&leftover, "untrusted edits").unwrap();
14539        let next = tmp.path().join("next-fixer");
14540        let mut failed = spec("first");
14541        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14542        let mut successor = spec("second");
14543        successor.command = vec![
14544            "sh".into(),
14545            "-c".into(),
14546            format!("touch '{}'; echo done", next.display()),
14547        ];
14548        let attempts = vec![
14549            (failed.clone(), "first".into()),
14550            (successor.clone(), "second".into()),
14551        ];
14552        let lock_path = runner.state.repo.join(".git/updater.lock");
14553        let external = std::fs::File::create(&lock_path).unwrap();
14554        assert_eq!(
14555            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14556            0
14557        );
14558        let artifacts = tmp.path().join("artifacts");
14559        let build = |spec, seat| SeatJob {
14560            spec,
14561            seat,
14562            cwd: winner.worktree.clone(),
14563            prompt: "fixture".into(),
14564            timeout: Duration::from_secs(10),
14565            allow_write: true,
14566            sessions: false,
14567            artifacts: artifacts.clone(),
14568            stem: "fix".into(),
14569            handover: None,
14570        };
14571        let (_, _, out) = runner
14572            .ask_fixer_among(
14573                attempts.clone(),
14574                "graft",
14575                None,
14576                Some((&winner.worktree, &head)),
14577                build,
14578            )
14579            .await;
14580        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14581        assert!(leftover.exists(), "contention must prevent clean");
14582        assert!(!next.exists());
14583        assert!(runner.state.handovers.is_empty());
14584        // Early interrupted-repair rollback uses the same protected route.
14585        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14586            from_run: "earlier".into(),
14587            generation: 1,
14588            mode: crate::recovery::Mode::VerifyRepair,
14589            reasons: Vec::new(),
14590            reviewed_head: None,
14591            rounds: Vec::new(),
14592            open_findings: Vec::new(),
14593            evidence: None,
14594            criteria_inherited: true,
14595            repair: crate::recovery::Repair::Started {
14596                before: head.clone(),
14597            },
14598        });
14599        assert!(
14600            runner
14601                .recovery_repair(&winner)
14602                .await
14603                .unwrap_err()
14604                .is::<crate::repo_lock::ResourceBlocked>()
14605        );
14606        assert!(matches!(
14607            runner.state.recovery.as_ref().unwrap().repair,
14608            crate::recovery::Repair::Started { .. }
14609        ));
14610        assert!(leftover.exists());
14611        drop(external);
14612        runner.recovery_repair(&winner).await.unwrap();
14613        assert!(!leftover.exists());
14614        assert!(matches!(
14615            runner.state.recovery.as_ref().unwrap().repair,
14616            crate::recovery::Repair::Done { .. }
14617        ));
14618        // A hard lock-open failure also leaves HEAD and untracked work intact.
14619        std::fs::remove_file(&lock_path).unwrap();
14620        std::fs::create_dir(&lock_path).unwrap();
14621        std::fs::write(&leftover, "untrusted edits").unwrap();
14622        let (_, _, out) = runner
14623            .ask_fixer_among(
14624                attempts,
14625                "graft",
14626                None,
14627                Some((&winner.worktree, &head)),
14628                build,
14629            )
14630            .await;
14631        assert!(matches!(out, AgentOutcome::Failed(_)));
14632        assert!(leftover.exists());
14633        assert!(!next.exists());
14634        assert_eq!(
14635            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14636            head
14637        );
14638    }
14639
14640    #[cfg(unix)]
14641    async fn registered_runner(root: &Path) -> Runner {
14642        repository_runner(root, crate::config::test_coordination()).await
14643    }
14644
14645    async fn repository_runner(
14646        root: &Path,
14647        coordination: Option<crate::config::Coordination>,
14648    ) -> Runner {
14649        ask_test_home();
14650        let repo = root.join("repo");
14651        std::fs::create_dir_all(&repo).unwrap();
14652        init_repo(&repo);
14653        if coordination.is_some() {
14654            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14655        }
14656        let mut runner = runner_at(RunStatus::Prep);
14657        runner.state.repo = repo.clone();
14658        runner.state.config.coordination = coordination;
14659        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14660        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14661        runner.state.config.graph.advise = false;
14662        runner.roles.implementers = vec![spec("implement")];
14663        runner.roles.judges = vec![spec("judge")];
14664        runner.prep().await.unwrap();
14665        runner
14666    }
14667
14668    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14669
14670    #[tokio::test]
14671    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14672        let tmp = tempfile::tempdir().unwrap();
14673        let mut runner = repository_runner(tmp.path(), None).await;
14674        let prompts = runner.state.config.prompts.clone();
14675        let run = runner.state.id.clone();
14676        let ctx = WaveCtx {
14677            carry_seats: false,
14678            pauses: None,
14679            run: &run,
14680            node: "advise",
14681            prompts: &prompts,
14682            cache: None,
14683            round: None,
14684        };
14685        let jobs: Vec<_> = ["alpha", "beta"]
14686            .into_iter()
14687            .map(|id| SeatJob {
14688                spec: scripted(id, AUTH_OUTAGE),
14689                seat: SeatState::new(id, id, 7),
14690                prompt: "propose".into(),
14691                cwd: runner.state.candidates[0].worktree.clone(),
14692                timeout: Duration::from_secs(10),
14693                allow_write: false,
14694                sessions: false,
14695                artifacts: agent::artifacts_dir(&runner.state.dir()),
14696                stem: id.into(),
14697                handover: None,
14698            })
14699            .collect();
14700        let mut losses = Vec::new();
14701        let results = ask_wave_with(
14702            jobs.clone(),
14703            runner.sem.clone(),
14704            2,
14705            &[],
14706            &ctx,
14707            &mut losses,
14708            &mut runner.state,
14709            &|_: &str| Ok(()),
14710        )
14711        .await;
14712        assert!(
14713            results
14714                .iter()
14715                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14716        );
14717        assert_eq!(runner.state.quota.len(), 2);
14718        assert!(
14719            runner
14720                .state
14721                .quota
14722                .iter()
14723                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14724        );
14725        assert!(!runner.state.advise_attempted);
14726        runner.stop_for_resource().unwrap();
14727        assert_eq!(runner.state.status, RunStatus::Blocked);
14728        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14729        assert_eq!(
14730            runner.state.resource_block.as_ref().unwrap().kind,
14731            "agent authentication"
14732        );
14733        // A machine loss followed by an ordinary failure is not evidence
14734        // that the whole node was unavailable, and must not earn a refund.
14735        runner.state.resource_block = None;
14736        runner.state.quota.clear();
14737        let mut mixed = jobs;
14738        mixed[1].spec.command = vec!["false".into()];
14739        let results = ask_wave_with(
14740            mixed,
14741            runner.sem.clone(),
14742            0,
14743            &[],
14744            &ctx,
14745            &mut losses,
14746            &mut runner.state,
14747            &|_: &str| Ok(()),
14748        )
14749        .await;
14750        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14751        assert!(runner.state.resource_block.is_none());
14752        assert_eq!(losses.len(), 1);
14753    }
14754
14755    #[tokio::test]
14756    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14757        let tmp = tempfile::tempdir().unwrap();
14758        let mut runner = repository_runner(tmp.path(), None).await;
14759        let implementer = scripted("implement", AUTH_OUTAGE);
14760        runner.roles.implementers = vec![implementer.clone()];
14761        runner.state.config.agents = vec![implementer];
14762        runner.implement().await.unwrap();
14763        assert_eq!(runner.state.status, RunStatus::Blocked);
14764        assert_eq!(runner.state.quota.len(), 1);
14765        assert_eq!(
14766            runner.state.quota[0].cause,
14767            agent::QuotaCause::Authentication
14768        );
14769        let candidate = &runner.state.candidates[0];
14770        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14771        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14772    }
14773
14774    #[tokio::test]
14775    #[cfg(unix)]
14776    async fn parked_implementer_persists_completed_work() {
14777        for registered in [false, true] {
14778            let tmp = tempfile::tempdir().unwrap();
14779            let mut runner = registered_runner(tmp.path()).await;
14780            if !registered {
14781                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14782            }
14783            let marker = tmp.path().join("started");
14784            let release = tmp.path().join("release");
14785            let mut implementer = spec("implement");
14786            implementer.command = vec![
14787                "sh".into(),
14788                "-c".into(),
14789                format!(
14790                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14791                    marker.display(),
14792                    release.display()
14793                ),
14794            ];
14795            runner.roles.implementers = vec![implementer.clone()];
14796            runner.state.config.agents = vec![implementer];
14797            let pause = runner.pause.clone();
14798            let (result, ()) = tokio::join!(runner.implement(), async {
14799                tokio::time::timeout(Duration::from_secs(10), async {
14800                    while !marker.exists() {
14801                        tokio::time::sleep(Duration::from_millis(10)).await;
14802                    }
14803                })
14804                .await
14805                .unwrap();
14806                pause.park();
14807                std::fs::write(&release, "go").unwrap();
14808            });
14809            result.unwrap();
14810            let candidate = &runner.state.candidates[0];
14811            assert!(candidate.commits > 0, "{candidate:?}");
14812            assert!(candidate.summary.contains("saved-result"));
14813            assert_eq!(candidate.files, 1);
14814            let before = candidate.commits;
14815            runner.implement().await.unwrap();
14816            assert_eq!(runner.state.candidates[0].commits, before);
14817        }
14818    }
14819
14820    #[tokio::test]
14821    #[cfg(unix)]
14822    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14823        let tmp = tempfile::tempdir().unwrap();
14824        let mut runner = registered_runner(tmp.path()).await;
14825        let winner = runner.state.candidates[0].clone();
14826        let cache = tmp.path().join("cache");
14827        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14828        runner.state.config.graph.timeout_resource_wait = 0;
14829        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14830        let crate::cache::AcquireOutcome::Acquired(held) =
14831            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14832        else {
14833            panic!("cache must be free")
14834        };
14835        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14836            from_run: "earlier".into(),
14837            generation: 1,
14838            mode: crate::recovery::Mode::VerifyRepair,
14839            reasons: Vec::new(),
14840            reviewed_head: None,
14841            rounds: Vec::new(),
14842            open_findings: Vec::new(),
14843            evidence: None,
14844            criteria_inherited: true,
14845            repair: crate::recovery::Repair::NotStarted,
14846        });
14847        runner.recovery_repair(&winner).await.unwrap();
14848        assert!(matches!(
14849            runner.state.recovery.as_ref().unwrap().repair,
14850            crate::recovery::Repair::WaitingForRestore { .. }
14851        ));
14852        drop(held);
14853        runner.state.resource_block = None;
14854        runner.state.config.graph.timeout_resource_wait = 1;
14855        runner.recovery_repair(&winner).await.unwrap();
14856        assert!(matches!(
14857            runner.state.recovery.as_ref().unwrap().repair,
14858            crate::recovery::Repair::Done { .. }
14859        ));
14860        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14861    }
14862
14863    #[tokio::test]
14864    #[cfg(unix)]
14865    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14866        use std::os::fd::AsRawFd as _;
14867        for graft in [false, true] {
14868            let tmp = tempfile::tempdir().unwrap();
14869            let mut runner = registered_runner(tmp.path()).await;
14870            runner.state.config.graph.timeout_resource_wait = 1;
14871            let winner = runner.state.candidates[0].clone();
14872            let marker = tmp.path().join("fixer-started");
14873            let mut fixer = spec("fixer");
14874            fixer.command = vec![
14875                "sh".into(),
14876                "-c".into(),
14877                format!("echo started >> '{}'; echo done", marker.display()),
14878            ];
14879            runner.roles.fixer = Some(vec![fixer]);
14880            if graft {
14881                let mut loser = winner.clone();
14882                loser.label = 'B';
14883                loser.index = 1;
14884                runner.state.candidates.push(loser);
14885                runner.state.config.graph.graft_runner_up = true;
14886                runner.state.tally = Some(Tally {
14887                    first_choice: BTreeMap::new(),
14888                    borda: BTreeMap::new(),
14889                    winner: winner.label,
14890                    rankings: 1,
14891                    unanimous_initial: true,
14892                    deliberated: false,
14893                    changed_votes: 0,
14894                    unanimous_final: true,
14895                    tie_break: None,
14896                    judges: 1,
14897                    present: 1,
14898                    quorum: 1,
14899                    met_quorum: true,
14900                    uncontested: None,
14901                });
14902                runner.state.judgements.push(crate::run::Judgement {
14903                    judge: 1,
14904                    seat: "judge-1".into(),
14905                    agent: "judge".into(),
14906                    ranking: vec![winner.label, 'B'],
14907                    reasons: BTreeMap::new(),
14908                    confidence: None,
14909                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14910                    order: vec![0, 1],
14911                    failed: None,
14912                    duration_ms: 0,
14913                });
14914            } else {
14915                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14916                    from_run: "earlier".into(),
14917                    generation: 1,
14918                    mode: crate::recovery::Mode::VerifyRepair,
14919                    reasons: Vec::new(),
14920                    reviewed_head: None,
14921                    rounds: Vec::new(),
14922                    open_findings: Vec::new(),
14923                    evidence: None,
14924                    criteria_inherited: true,
14925                    repair: crate::recovery::Repair::NotStarted,
14926                });
14927            }
14928            let reader = crate::repo_lock::SeatLease::for_checkout(
14929                &winner.worktree,
14930                false,
14931                crate::config::test_coordination().as_ref(),
14932            )
14933            .await
14934            .unwrap();
14935            let pause = runner.pause.clone();
14936            let (result, ()) = tokio::join!(
14937                async {
14938                    if graft {
14939                        runner.graft().await
14940                    } else {
14941                        runner.recovery_repair(&winner).await
14942                    }
14943                },
14944                async {
14945                    tokio::time::sleep(Duration::from_millis(100)).await;
14946                    pause.park();
14947                }
14948            );
14949            result.unwrap();
14950            assert!(!marker.exists(), "a blocked seat must not spawn");
14951            // Exercise the persisted state, as a binary replacement/resume does.
14952            runner.state = RunState::load(&runner.state.id).unwrap();
14953            if graft {
14954                assert_eq!(
14955                    runner.state.graft.as_ref().unwrap().outcome,
14956                    GraftOutcome::WaitingForRestore
14957                );
14958            } else {
14959                assert!(matches!(
14960                    runner.state.recovery.as_ref().unwrap().repair,
14961                    crate::recovery::Repair::WaitingForRestore { .. }
14962                ));
14963            }
14964            drop(reader);
14965            runner.pause = Pause::new();
14966            runner.state.resource_block = None;
14967            // Deferred restoration must still respect an external lock owner.
14968            let external =
14969                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14970            assert_eq!(
14971                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14972                0
14973            );
14974            let leftover = winner.worktree.join("leftover");
14975            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14976            let result = if graft {
14977                runner.graft().await
14978            } else {
14979                runner.recovery_repair(&winner).await
14980            };
14981            assert!(
14982                result
14983                    .unwrap_err()
14984                    .is::<crate::repo_lock::ResourceBlocked>()
14985            );
14986            assert!(leftover.exists());
14987            assert!(!marker.exists());
14988            drop(external);
14989            runner.state.resource_block = None;
14990            let result = if graft {
14991                runner.graft().await
14992            } else {
14993                runner.recovery_repair(&winner).await
14994            };
14995            result.unwrap();
14996            assert!(!leftover.exists(), "restore must precede retry");
14997            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14998            if graft {
14999                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15000            } else {
15001                assert!(matches!(
15002                    runner.state.recovery.as_ref().unwrap().repair,
15003                    crate::recovery::Repair::Done { .. }
15004                ));
15005            }
15006        }
15007    }
15008
15009    #[tokio::test]
15010    #[cfg(unix)]
15011    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15012        use std::os::fd::AsRawFd as _;
15013        let tmp = tempfile::tempdir().unwrap();
15014        let mut runner = registered_runner(tmp.path()).await;
15015        runner.state.config.graph.timeout_resource_wait = 1;
15016        let winner = runner.state.candidates[0].clone();
15017        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15018        runner.state.tally = Some(Tally {
15019            first_choice: BTreeMap::new(),
15020            borda: BTreeMap::new(),
15021            winner: winner.label,
15022            rankings: 0,
15023            unanimous_initial: false,
15024            deliberated: false,
15025            changed_votes: 0,
15026            unanimous_final: false,
15027            tie_break: None,
15028            judges: 0,
15029            present: 0,
15030            quorum: 0,
15031            met_quorum: true,
15032            uncontested: Some("fixture".into()),
15033        });
15034        runner.state.graft = Some(GraftRecord {
15035            pre: pre.clone(),
15036            post: None,
15037            lines: None,
15038            limit: None,
15039            agent: None,
15040            outcome: GraftOutcome::Running,
15041            requested: Vec::new(),
15042            adoption: None,
15043            adoption_error: None,
15044        });
15045        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15046        git::commit_all(&winner.worktree, "unfinished graft")
15047            .await
15048            .unwrap();
15049        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15050        let leftover = winner.worktree.join("leftover");
15051        std::fs::write(&leftover, "untrusted edits").unwrap();
15052        let path = runner.state.repo.join(".git/updater.lock");
15053        let external = std::fs::File::create(&path).unwrap();
15054        assert_eq!(
15055            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15056            0
15057        );
15058        assert!(
15059            runner
15060                .graft()
15061                .await
15062                .unwrap_err()
15063                .is::<crate::repo_lock::ResourceBlocked>()
15064        );
15065        assert!(matches!(
15066            runner.state.graft.as_ref().unwrap().outcome,
15067            GraftOutcome::Running
15068        ));
15069        assert_eq!(
15070            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15071            unfinished
15072        );
15073        assert!(leftover.exists());
15074        drop(external);
15075        std::fs::remove_file(&path).unwrap();
15076        std::fs::create_dir(&path).unwrap();
15077        assert!(runner.graft().await.is_err());
15078        assert!(matches!(
15079            runner.state.graft.as_ref().unwrap().outcome,
15080            GraftOutcome::Running
15081        ));
15082        assert_eq!(
15083            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15084            unfinished
15085        );
15086        assert!(leftover.exists());
15087        std::fs::remove_dir(&path).unwrap();
15088        runner.graft().await.unwrap();
15089        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15090        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15091        assert!(!leftover.exists());
15092    }
15093
15094    #[tokio::test]
15095    #[cfg(unix)]
15096    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15097        use std::os::fd::AsRawFd as _;
15098        let tmp = tempfile::tempdir().unwrap();
15099        let mut runner = registered_runner(tmp.path()).await;
15100        runner.state.prep_complete = Some(false);
15101        runner.state.config.graph.timeout_resource_wait = 1;
15102        runner.state.save().unwrap();
15103        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15104        assert_eq!(
15105            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15106            0
15107        );
15108        runner.execute().await.unwrap();
15109        let disk = RunState::load(&runner.state.id).unwrap();
15110        assert!(disk.driver_exited);
15111        assert_eq!(disk.driver_pid, None);
15112        assert_eq!(runner.state.status, RunStatus::Blocked);
15113        assert_eq!(
15114            runner.state.resource_block.as_ref().unwrap().kind,
15115            "repository coordination"
15116        );
15117        assert!(runner.state.prep_failure.is_none());
15118        assert!(runner.state.seats.is_empty());
15119        runner.pause.park();
15120        // Include headroom for Git subprocesses under parallel test load;
15121        // the repository contention deadline itself remains one second.
15122        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15123            .await
15124            .unwrap()
15125            .unwrap();
15126        assert!(runner.state.parked);
15127        assert!(runner.state.prep_failure.is_none());
15128        let disk = RunState::load(&runner.state.id).unwrap();
15129        assert!(disk.driver_exited);
15130        assert_eq!(disk.driver_pid, None);
15131    }
15132
15133    #[tokio::test]
15134    #[cfg(unix)]
15135    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15136        use std::os::fd::AsRawFd as _;
15137        let tmp = tempfile::tempdir().unwrap();
15138        let mut runner = registered_runner(tmp.path()).await;
15139        runner.state.prep_complete = Some(false);
15140        runner.state.config.graph.timeout_resource_wait = 30;
15141        runner.state.save().unwrap();
15142        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15143        assert_eq!(
15144            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15145            0
15146        );
15147        assert!(
15148            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15149                .await
15150                .is_err()
15151        );
15152        let disk = RunState::load(&runner.state.id).unwrap();
15153        assert!(disk.driver_exited);
15154        assert_eq!(disk.driver_pid, None);
15155        assert!(disk.status.resumable());
15156        assert!(runner.state.driver_exited);
15157        assert_eq!(runner.state.driver_pid, None);
15158        runner.state.save().unwrap();
15159        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15160    }
15161
15162    #[tokio::test]
15163    #[cfg(unix)]
15164    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15165        let tmp = tempfile::tempdir().unwrap();
15166        let mut runner = registered_runner(tmp.path()).await;
15167        let winner = runner.state.candidates[0].clone();
15168        let cache = tmp.path().join("cache");
15169        let started = tmp.path().join("pre-gate-started");
15170        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15171        runner.state.config.verify.gate =
15172            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15173        let home = crate::run::home();
15174        let owner =
15175            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15176        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15177            crate::cache::AcquireOutcome::Acquired(g) => g,
15178            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15179        };
15180        let mut gate = Box::pin(runner.run_gate(&winner));
15181        assert!(
15182            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15183                .await
15184                .is_err()
15185        );
15186        assert!(!started.exists(), "pre-gate must wait for the cache too");
15187        // Simulate the wave that already owns the cache entering invoke.
15188        // It must obtain the editor reservation and finish without waiting
15189        // for the gate that is queued behind its cache.
15190        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15191        let editor = crate::repo_lock::SeatLease::checkout_with(
15192            &winner.worktree,
15193            true,
15194            crate::config::test_coordination().as_ref(),
15195            &wait,
15196        )
15197        .await
15198        .unwrap();
15199        drop(editor);
15200        guard.release();
15201        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15202            .await
15203            .unwrap()
15204            .unwrap();
15205        drop(gate);
15206        assert!(outcomes.iter().all(CommandOutcome::ok));
15207        assert!(started.exists());
15208    }
15209
15210    #[tokio::test]
15211    #[cfg(unix)]
15212    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15213        use std::os::fd::AsRawFd as _;
15214
15215        ask_test_home();
15216        let tmp = tempfile::tempdir().unwrap();
15217        let repo = tmp.path().join("repo");
15218        std::fs::create_dir_all(&repo).unwrap();
15219        init_repo(&repo);
15220        let common = repo.join(".git");
15221        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15222        let mut runner = runner_at(RunStatus::Prep);
15223        runner.state.repo = repo.clone();
15224        runner.state.config.coordination = crate::config::test_coordination();
15225        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15226        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15227        runner.state.config.graph.advise = false;
15228        runner.roles.implementers = vec![spec("implement")];
15229        runner.roles.judges = vec![spec("judge")];
15230        runner.prep().await.unwrap();
15231        let winner = runner.state.candidates[0].clone();
15232        // Emulate edits already left by a completed seat. The fixture CLI
15233        // only reports completion; it performs no repository operations.
15234        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15235        let completed = tmp.path().join("seat-completed");
15236        runner.roles.implementers[0].command = vec![
15237            "sh".into(),
15238            "-c".into(),
15239            format!("touch '{}'; echo completed", completed.display()),
15240        ];
15241        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15242        assert_eq!(
15243            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15244            0
15245        );
15246        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15247        let mut implement = Box::pin(runner.implement());
15248        tokio::select! {
15249            result = &mut implement => panic!("rescue must wait: {result:?}"),
15250            result = tokio::time::timeout(Duration::from_secs(10), async {
15251                while !completed.exists() {
15252                    tokio::time::sleep(Duration::from_millis(20)).await;
15253                }
15254            }) => result.unwrap(),
15255        }
15256        assert!(
15257            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15258                .await
15259                .is_err()
15260        );
15261        assert_eq!(
15262            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15263            before
15264        );
15265        assert_eq!(
15266            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15267            0
15268        );
15269        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15270            .await
15271            .unwrap()
15272            .unwrap();
15273        drop(implement);
15274        assert_ne!(
15275            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15276            before
15277        );
15278        assert!(git::is_clean(&winner.worktree).await.unwrap());
15279
15280        let started = tmp.path().join("pre-gate-started");
15281        let proceed = tmp.path().join("pre-gate-proceed");
15282        let checked = tmp.path().join("gate-ran");
15283        runner.state.config.verify.pre_gate = vec![format!(
15284            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15285            started.display(),
15286            proceed.display()
15287        )];
15288        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15289        assert_eq!(
15290            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15291            0
15292        );
15293        let mut gate = Box::pin(runner.run_gate(&winner));
15294        assert!(
15295            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15296                .await
15297                .is_err()
15298        );
15299        assert!(!started.exists());
15300        assert!(!checked.exists());
15301        assert!(git::is_clean(&winner.worktree).await.unwrap());
15302        assert_eq!(
15303            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15304            0
15305        );
15306        tokio::select! {
15307            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15308            result = tokio::time::timeout(Duration::from_secs(10), async {
15309                while !started.exists() {
15310                    tokio::time::sleep(Duration::from_millis(20)).await;
15311                }
15312            }) => result.unwrap(),
15313        }
15314        assert_eq!(
15315            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15316            -1,
15317            "mechanical edits must hold the real lock"
15318        );
15319        std::fs::write(proceed, "ready").unwrap();
15320        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15321            .await
15322            .unwrap()
15323            .unwrap();
15324        drop(gate);
15325        assert!(outcomes.iter().all(CommandOutcome::ok));
15326        assert!(checked.exists());
15327        assert!(runner.state.pre_gate_commit.is_some());
15328        assert!(git::is_clean(&winner.worktree).await.unwrap());
15329        assert_eq!(
15330            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15331            0,
15332            "gate completion must release coordination"
15333        );
15334        drop(outsider);
15335        std::fs::remove_file(common.join("updater.lock")).unwrap();
15336        std::fs::create_dir(common.join("updater.lock")).unwrap();
15337        std::fs::remove_file(&started).unwrap();
15338        std::fs::remove_file(&checked).unwrap();
15339        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15340        let error = runner.run_gate(&winner).await.unwrap_err();
15341        assert!(
15342            error
15343                .to_string()
15344                .contains("acquire mandatory repository lock")
15345        );
15346        assert!(
15347            !started.exists(),
15348            "failed acquisition must prevent pre-gate"
15349        );
15350        assert!(!checked.exists(), "failed acquisition must prevent gate");
15351        assert!(git::is_clean(&winner.worktree).await.unwrap());
15352        assert_eq!(
15353            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15354            head
15355        );
15356    }
15357
15358    #[tokio::test]
15359    #[cfg(unix)]
15360    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15361        use std::os::fd::AsRawFd as _;
15362        use std::time::Duration;
15363
15364        ask_test_home();
15365        let tmp = tempfile::tempdir().unwrap();
15366        let repo = tmp.path().join("repo");
15367        std::fs::create_dir_all(&repo).unwrap();
15368        init_repo(&repo);
15369        let common = repo.join(".git");
15370        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15371        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15372        assert_eq!(
15373            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15374            0
15375        );
15376        let mut runner = runner_at(RunStatus::Prep);
15377        runner.state.repo = repo.clone();
15378        runner.state.config.coordination = crate::config::test_coordination();
15379        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15380        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15381        runner.state.config.graph.advise = true;
15382        runner.state.config.graph.advisors = 1;
15383        runner.roles.implementers = vec![spec("implement")];
15384        runner.roles.judges = vec![spec("judge")];
15385        let root = runner.state.worktree_root();
15386        let started = tmp.path().join("setup-started");
15387        let proceed = tmp.path().join("setup-proceed");
15388        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15389            run: Some(format!(
15390                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15391                started.display(),
15392                proceed.display()
15393            )),
15394            ..Default::default()
15395        }];
15396        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15397        let config = git::git(&repo, &["config", "--local", "--list"])
15398            .await
15399            .unwrap();
15400        let mut prep = Box::pin(runner.prep());
15401        assert!(
15402            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15403                .await
15404                .is_err()
15405        );
15406        assert!(
15407            !root.exists(),
15408            "contention must prevent even checkout creation"
15409        );
15410        assert!(!started.exists(), "contention must prevent setup");
15411        assert_eq!(
15412            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15413            branches
15414        );
15415        assert_eq!(
15416            git::git(&repo, &["config", "--local", "--list"])
15417                .await
15418                .unwrap(),
15419            config
15420        );
15421        drop(lock);
15422        tokio::select! {
15423            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15424            result = tokio::time::timeout(Duration::from_secs(10), async {
15425                while !started.exists() {
15426                    tokio::time::sleep(Duration::from_millis(20)).await;
15427                }
15428            }) => result.unwrap(),
15429        }
15430        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15431        assert_eq!(
15432            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15433            -1
15434        );
15435        assert!(
15436            tokio::time::timeout(
15437                Duration::from_millis(100),
15438                crate::repo_lock::SeatLease::for_checkout(
15439                    &repo,
15440                    true,
15441                    crate::config::test_coordination().as_ref()
15442                )
15443            )
15444            .await
15445            .is_err(),
15446            "preparation must also exclude local editors"
15447        );
15448        std::fs::write(proceed, "ready").unwrap();
15449        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15450            .await
15451            .unwrap()
15452            .unwrap();
15453        drop(prep);
15454        assert_eq!(runner.state.prep_complete, Some(true));
15455        assert!(root.join("judge-1").exists());
15456        assert!(root.join("advisor-1").exists());
15457        assert_eq!(
15458            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15459            0
15460        );
15461    }
15462
15463    #[tokio::test]
15464    #[cfg(unix)]
15465    async fn preparation_lock_failure_prevents_repository_changes() {
15466        ask_test_home();
15467        let tmp = tempfile::tempdir().unwrap();
15468        let repo = tmp.path().join("repo");
15469        std::fs::create_dir_all(&repo).unwrap();
15470        init_repo(&repo);
15471        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15472        // A directory at the required lock path makes opening the real lock
15473        // fail, rather than merely exercising an invalid registration.
15474        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15475        let mut runner = runner_at(RunStatus::Prep);
15476        runner.state.repo = repo.clone();
15477        runner.state.config.coordination = crate::config::test_coordination();
15478        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15479        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15480        runner.roles.implementers = vec![spec("implement")];
15481        let error = runner.prep().await.unwrap_err();
15482        assert!(
15483            error
15484                .to_string()
15485                .contains("acquire mandatory repository lock")
15486        );
15487        assert!(!runner.state.worktree_root().exists());
15488        assert!(runner.state.candidates.is_empty());
15489        assert!(!repo.join(".git/config.worktree").exists());
15490        assert_eq!(
15491            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15492            "* main"
15493        );
15494    }
15495
15496    #[tokio::test]
15497    async fn preparation_records_creation_intent_before_git_can_fail() {
15498        ask_test_home();
15499        let tmp = tempfile::tempdir().unwrap();
15500        let repo = tmp.path().join("repo");
15501        std::fs::create_dir_all(&repo).unwrap();
15502        init_repo(&repo);
15503        let mut runner = runner_at(RunStatus::Prep);
15504        runner.state.repo = repo.clone();
15505        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15506        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15507        runner.state.config.blind.commit_msg_hook = false;
15508        runner.roles.implementers = vec![spec("one")];
15509        let label = blind::assign_labels(1, runner.state.seed)[0];
15510        let branch = runner.state.branch_for(label);
15511        // A conflicting branch makes Git refuse creation before a candidate
15512        // can be registered. The durable intent must still identify it.
15513        git::git(&repo, &["branch", &branch]).await.unwrap();
15514        assert!(runner.prep().await.is_err());
15515        let saved = RunState::load(&runner.state.id).unwrap();
15516        assert!(saved.candidates.is_empty());
15517        let event = saved
15518            .events
15519            .iter()
15520            .find(|e| e.node == "prep-intent")
15521            .unwrap();
15522        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15523        assert_eq!(intent["branch"], branch);
15524        assert_eq!(intent["start"], runner.state.base_commit);
15525        assert_eq!(
15526            intent["worktree"],
15527            runner
15528                .state
15529                .worktree_root()
15530                .join(format!("cand-{label}"))
15531                .to_string_lossy()
15532                .as_ref()
15533        );
15534        assert_eq!(intent["picks"], serde_json::json!([]));
15535    }
15536
15537    #[tokio::test]
15538    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15539        ask_test_home();
15540        let tmp = tempfile::tempdir().unwrap();
15541        let repo = tmp.path().join("repo");
15542        std::fs::create_dir_all(&repo).unwrap();
15543        init_repo(&repo);
15544        let mut runner = runner_at(RunStatus::Prep);
15545        runner.state.repo = repo.clone();
15546        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15547        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15548        runner.state.sources.pick_commits.push(refs::Input {
15549            token: "missing".into(),
15550            sha: "0".repeat(40),
15551            provenance: refs::Provenance::ExplicitCli,
15552        });
15553        assert!(runner.prep().await.is_err());
15554        assert!(runner.state.candidates.is_empty());
15555        assert!(!runner.state.worktree_root().exists());
15556    }
15557
15558    #[tokio::test]
15559    async fn prose_and_legacy_inference_never_import_into_candidates() {
15560        ask_test_home();
15561        let tmp = tempfile::tempdir().unwrap();
15562        let repo = tmp.path().join("repo");
15563        std::fs::create_dir_all(&repo).unwrap();
15564        init_repo(&repo);
15565        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15566        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15567            .await
15568            .unwrap();
15569        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15570        git::git(&repo, &["add", "."]).await.unwrap();
15571        git::git(&repo, &["commit", "-m", "historical implementation"])
15572            .await
15573            .unwrap();
15574        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15575        git::git(&repo, &["checkout", "main"]).await.unwrap();
15576        for instruction in [
15577            format!("Example: magi/history/A at {source}"),
15578            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15579            format!("Never import magi/history/A or {source}"),
15580            format!("Conversation answer: I mentioned {source} only as background"),
15581        ] {
15582            let mut runner = runner_at(RunStatus::Prep);
15583            runner.state.repo = repo.clone();
15584            runner.state.base_commit = base.clone();
15585            runner.state.instruction = instruction.clone();
15586            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15587            runner.state.config.blind.commit_msg_hook = false;
15588            runner.state.seeds = vec![refs::Seed {
15589                token: "magi/history/A".into(),
15590                sha: source.clone(),
15591                kind: refs::SeedKind::Unmerged,
15592                branch: true,
15593                detail: String::new(),
15594            }];
15595            runner.roles.implementers = vec![spec("one")];
15596            assert_eq!(seeded_instruction(&runner.state), instruction);
15597            // Model an old record with inferred seeds but no structured input field.
15598            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15599            legacy.as_object_mut().unwrap().remove("sources");
15600            runner.state = serde_json::from_value(legacy).unwrap();
15601            assert_eq!(runner.state.sources, refs::Sources::default());
15602            runner.state.save().unwrap();
15603            runner.state = RunState::load(&runner.state.id).unwrap();
15604            runner.prep().await.unwrap();
15605            for candidate in &runner.state.candidates {
15606                assert_eq!(
15607                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15608                    base
15609                );
15610                assert!(!candidate.worktree.join("unwanted").exists());
15611            }
15612        }
15613    }
15614
15615    #[tokio::test]
15616    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15617        ask_test_home();
15618        let tmp = tempfile::tempdir().unwrap();
15619        let repo = tmp.path().join("repo");
15620        std::fs::create_dir_all(&repo).unwrap();
15621        init_repo(&repo);
15622        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15623        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15624        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15625        git::git(&repo, &["add", "."]).await.unwrap();
15626        git::git(&repo, &["commit", "-m", "required seed"])
15627            .await
15628            .unwrap();
15629        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15630        git::git(&repo, &["checkout", "main"]).await.unwrap();
15631        let mut runner = runner_at(RunStatus::Prep);
15632        runner.state.repo = repo.clone();
15633        runner.state.base_commit = base.clone();
15634        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15635        runner.state.config.graph.advise = false;
15636        runner.state.config.blind.commit_msg_hook = false;
15637        runner.state.sources =
15638            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15639                .await
15640                .unwrap();
15641        runner.roles.implementers = vec![spec("one"), spec("two")];
15642        runner.roles.judges = vec![spec("judge")];
15643        let label = blind::assign_labels(2, runner.state.seed)[0];
15644        let branch = runner.state.branch_for(label);
15645        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15646        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15647            .await
15648            .unwrap();
15649        runner.state.candidates.push(Candidate {
15650            index: 0,
15651            label,
15652            agent: "one".into(),
15653            branch,
15654            worktree,
15655            summary: String::new(),
15656            stat: String::new(),
15657            files: 0,
15658            commits: 0,
15659            empty: false,
15660            failed: None,
15661            verified_noop: None,
15662            duration_ms: 0,
15663            folded: false,
15664        });
15665        runner.state.save().unwrap();
15666        // Reload the checkpoint immediately before its first cherry-pick.
15667        runner.state = RunState::load(&runner.state.id).unwrap();
15668        assert_eq!(runner.state.prep_complete, Some(false));
15669        runner.prep().await.unwrap();
15670        assert_eq!(runner.state.prep_complete, Some(true));
15671        assert_eq!(runner.state.candidates.len(), 2);
15672        for candidate in &runner.state.candidates {
15673            assert_eq!(
15674                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15675                git::tree_of(&repo, &seed).await.unwrap()
15676            );
15677            assert_eq!(
15678                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15679                "required commit"
15680            );
15681        }
15682        assert!(runner.state.worktree_root().join("judge-1").exists());
15683        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15684        runner.prep().await.unwrap();
15685        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15686
15687        runner.state.prep_complete = Some(false);
15688        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15689            run: Some(format!(
15690                "test -f '{}'",
15691                tmp.path().join("dependency-ready").display()
15692            )),
15693            ..Default::default()
15694        }];
15695        assert!(runner.prep().await.is_err());
15696        assert!(runner.state.prep_failure.is_some());
15697        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15698        runner.state = RunState::load(&runner.state.id).unwrap();
15699        assert!(
15700            runner.prep().await.is_err(),
15701            "correcting the cause alone must not auto-retry"
15702        );
15703        let failed_at = runner.state.events.last().unwrap().at;
15704        assert!(
15705            runner.retry_preparation(failed_at).is_err(),
15706            "old approvals cannot unlock the hold"
15707        );
15708        let mut task = crate::queue::Task::new(
15709            "retry".into(),
15710            "retry setup".into(),
15711            repo.clone(),
15712            crate::queue::Source::Human,
15713        );
15714        task.start(runner.state.id.clone());
15715        task.hold_machine(Some("setup failed".into()));
15716        task.release_operator("cli", None);
15717        runner
15718            .retry_preparation(task.resumption.as_ref().unwrap().at)
15719            .unwrap();
15720        runner.prep().await.unwrap();
15721        assert!(runner.state.prep_failure.is_none());
15722        assert_eq!(runner.state.prep_complete, Some(true));
15723        assert_eq!(runner.state.candidates.len(), 2);
15724        assert!(runner.state.worktree_root().join("judge-1").exists());
15725        for candidate in &runner.state.candidates {
15726            assert_eq!(
15727                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15728                git::tree_of(&repo, &seed).await.unwrap()
15729            );
15730        }
15731    }
15732
15733    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15734        let mut heads = Vec::new();
15735        for candidate in candidates {
15736            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15737        }
15738        heads
15739    }
15740
15741    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15742        ask_test_home();
15743        let tmp = tempfile::tempdir().unwrap();
15744        let repo = tmp.path().join("repo");
15745        std::fs::create_dir(&repo).unwrap();
15746        init_repo(&repo);
15747        let mut runner = runner_at(RunStatus::Reviewing);
15748        runner.state.repo = repo.clone();
15749        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15750        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15751        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15752            let branch = format!("candidate-{label}");
15753            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15754            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15755                .await
15756                .unwrap();
15757            if label == 'B' {
15758                std::fs::write(worktree.join("change"), "winner").unwrap();
15759                git::rescue_commit(&worktree, "candidate change")
15760                    .await
15761                    .unwrap();
15762            }
15763            runner.state.candidates.push(Candidate {
15764                index,
15765                label,
15766                agent: "alpha".into(),
15767                branch,
15768                worktree,
15769                summary: "completed reply".into(),
15770                stat: String::new(),
15771                files: usize::from(label == 'B'),
15772                commits: usize::from(label == 'B'),
15773                empty: label == 'A',
15774                failed: None,
15775                verified_noop: None,
15776                duration_ms: 1,
15777                folded: false,
15778            });
15779        }
15780        runner.state.judge_skipped = true;
15781        runner.state.tally = Some(Tally {
15782            first_choice: BTreeMap::new(),
15783            borda: BTreeMap::new(),
15784            winner: 'B',
15785            rankings: 0,
15786            unanimous_initial: false,
15787            deliberated: false,
15788            changed_votes: 0,
15789            unanimous_final: false,
15790            tie_break: None,
15791            judges: 0,
15792            present: 0,
15793            quorum: 0,
15794            met_quorum: true,
15795            uncontested: Some("only B produced a change".into()),
15796        });
15797        runner.state.save().unwrap();
15798        (tmp, runner)
15799    }
15800
15801    #[tokio::test]
15802    #[cfg(unix)]
15803    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15804        for (solo, failed_checkpoint) in
15805            [(true, false), (false, false), (true, true), (false, true)]
15806        {
15807            let (_tmp, mut runner) = preservation_resume_fixture().await;
15808            if solo {
15809                runner.state.candidates.retain(|c| c.label == 'A');
15810            }
15811            runner.state.candidates[0].empty = false;
15812            runner.state.candidates[0].summary.clear();
15813            runner.state.tally = None;
15814            runner.state.judge_skipped = false;
15815            runner.state.status = RunStatus::Implementing;
15816            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15817            let mut implementer = spec("alpha");
15818            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15819            runner.roles.implementers = vec![implementer];
15820            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15821            if failed_checkpoint {
15822                std::fs::create_dir_all(&patch).unwrap();
15823            }
15824            runner.state.save().unwrap();
15825            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15826            drop(pressure);
15827            runner.state = RunState::load(&runner.state.id).unwrap();
15828            assert_eq!(runner.state.status, RunStatus::Blocked);
15829            assert!(runner.state.interrupted_implementations.contains(&'A'));
15830            assert_eq!(runner.state.candidates[0].commits, 0);
15831            assert!(!runner.state.candidates[0].empty);
15832            assert!(runner.state.candidates[0].failed.is_none());
15833            if failed_checkpoint {
15834                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15835                std::fs::remove_dir(patch).unwrap();
15836            }
15837            runner.state.resource_block = None;
15838            runner.state.config.disk.min_free_bytes = 0;
15839            runner.roles.implementers[0].command = vec![
15840                "sh".into(),
15841                "-c".into(),
15842                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15843            ];
15844            runner.implement().await.unwrap();
15845            assert!(runner.state.interrupted_implementations.is_empty());
15846            assert!(runner.state.candidate_preservation.is_empty());
15847            assert_eq!(runner.state.candidates[0].commits, 1);
15848            assert!(!runner.state.candidates[0].empty);
15849            assert!(
15850                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15851                    .unwrap()
15852                    .contains("retried empty interrupted seat")
15853            );
15854            if !solo {
15855                assert_eq!(runner.state.candidates[1].commits, 1);
15856            }
15857        }
15858    }
15859
15860    #[tokio::test]
15861    #[cfg(unix)]
15862    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15863        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15864            let (_tmp, mut runner) = preservation_resume_fixture().await;
15865            runner.state.candidates.retain(|c| c.label == 'A');
15866            runner.state.candidates[0].empty = false;
15867            runner.state.candidates[0].summary.clear();
15868            runner.state.tally = None;
15869            runner.state.judge_skipped = false;
15870            runner.state.status = RunStatus::Implementing;
15871            // Force only the real post-spawn watchdog, without failing the
15872            // separate admission check. Existing edits predate its measurement.
15873            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15874            let wt = runner.state.candidates[0].worktree.clone();
15875            std::fs::write(
15876                wt.join("interrupted-source"),
15877                "retain implementation work\n",
15878            )
15879            .unwrap();
15880            if committed {
15881                git::rescue_commit(&wt, "implementation checkpoint")
15882                    .await
15883                    .unwrap();
15884            }
15885            let lock = git::git(
15886                &wt,
15887                &[
15888                    "rev-parse",
15889                    "--path-format=absolute",
15890                    "--git-path",
15891                    "index.lock",
15892                ],
15893            )
15894            .await
15895            .unwrap();
15896            if block_rescue {
15897                std::fs::write(&lock, "fixture-owned lock").unwrap();
15898            }
15899            let mut implementer = spec("alpha");
15900            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15901            runner.roles.implementers = vec![implementer];
15902            runner.state.save().unwrap();
15903            let result = runner.implement().await;
15904            drop(pressure);
15905            assert_eq!(result.is_err(), block_rescue);
15906            assert_eq!(
15907                runner.state.resource_block.as_ref().unwrap().kind,
15908                "disk pressure"
15909            );
15910            assert!(
15911                runner
15912                    .state
15913                    .events
15914                    .iter()
15915                    .any(|e| e.message.contains("agent execution interrupted"))
15916            );
15917            assert!(
15918                runner
15919                    .state
15920                    .dir()
15921                    .join("artifacts/candidate-A-recovery/staged.patch")
15922                    .exists()
15923            );
15924            if block_rescue {
15925                let saved = RunState::load(&runner.state.id).unwrap();
15926                assert!(saved.candidate_preservation.contains_key(&'A'));
15927                assert!(!saved.candidates[0].empty);
15928                assert_eq!(
15929                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15930                    "retain implementation work\n"
15931                );
15932                std::fs::remove_file(lock).unwrap();
15933                runner.state = saved;
15934            } else {
15935                assert_eq!(runner.state.status, RunStatus::Blocked);
15936                assert_eq!(runner.state.candidates[0].commits, 1);
15937            }
15938            // Recovery must save the candidate before deciding whether another
15939            // implementation is needed. No implementer is available on retry.
15940            runner.roles.implementers.clear();
15941            runner.state.config.disk.min_free_bytes = 0;
15942            runner.state.resource_block = None;
15943            runner.implement().await.unwrap();
15944            assert!(runner.state.candidate_preservation.is_empty());
15945            assert_eq!(runner.state.candidates[0].commits, 1);
15946            assert!(!runner.state.candidates[0].empty);
15947            assert!(git::is_clean(&wt).await.unwrap());
15948            assert!(
15949                crate::run::read_artifact(&runner.state, "cand-A.patch")
15950                    .unwrap()
15951                    .contains("retain implementation work")
15952            );
15953        }
15954    }
15955
15956    #[tokio::test]
15957    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15958        let (_tmp, mut runner) = preservation_resume_fixture().await;
15959        // Schema 15 already confirmed the unfolded no-change result.
15960        runner.implement().await.unwrap();
15961        assert!(runner.state.candidate_preservation.is_empty());
15962        assert!(
15963            !runner
15964                .state
15965                .dir()
15966                .join("artifacts/candidate-A-recovery")
15967                .exists()
15968        );
15969        runner.fold_losers().await.unwrap();
15970        assert!(!runner.state.candidates[0].worktree.exists());
15971        for schema in [14, crate::run::SCHEMA] {
15972            runner.state.schema = schema;
15973            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15974            runner.state.save().unwrap();
15975            runner.state = RunState::load(&runner.state.id).unwrap();
15976            runner.implement().await.unwrap();
15977            assert!(runner.state.candidate_preservation.is_empty());
15978            assert!(runner.state.candidates[0].folded);
15979            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15980        }
15981    }
15982
15983    #[tokio::test]
15984    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15985        let (_tmp, mut runner) = preservation_resume_fixture().await;
15986        runner.state.candidates.retain(|c| c.label == 'A');
15987        runner.state.tally = None;
15988        runner.state.judge_skipped = false;
15989        let candidate = &mut runner.state.candidates[0];
15990        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15991        candidate.failed = Some("deployment validation unavailable".into());
15992        std::fs::write(
15993            candidate.worktree.join("repair"),
15994            "future launcher policy\n",
15995        )
15996        .unwrap();
15997        runner
15998            .state
15999            .candidate_preservation
16000            .insert('A', "awaiting persistence".into());
16001        runner.persist_candidates().await.unwrap();
16002        let candidate = &runner.state.candidates[0];
16003        assert!(!candidate.empty);
16004        assert_eq!(candidate.commits, 1);
16005        assert_eq!(candidate.files, 1);
16006        assert!(candidate.failed.is_none());
16007        assert!(candidate.summary.contains("EPERM"));
16008        assert!(
16009            crate::run::read_artifact(&runner.state, "cand-A.patch")
16010                .unwrap()
16011                .contains("future launcher policy")
16012        );
16013        assert!(runner.state.candidate_preservation.is_empty());
16014    }
16015
16016    #[tokio::test]
16017    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16018        let (_tmp, mut runner) = preservation_resume_fixture().await;
16019        let wt = runner.state.candidates[0].worktree.clone();
16020        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16021        runner.state.schema = 14;
16022        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16023        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16024        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16025        for _ in 0..2 {
16026            assert!(runner.implement().await.is_err());
16027            assert_eq!(runner.state.status, RunStatus::Blocked);
16028            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16029            assert!(
16030                runner.state.candidates[0].empty,
16031                "old verdict must remain unusable"
16032            );
16033            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16034            assert!(runner.state.judge_skipped);
16035            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16036            assert!(
16037                crate::run::read_artifact(&runner.state, "cand-A.patch")
16038                    .unwrap()
16039                    .contains("owned edits")
16040            );
16041            assert!(wt.exists());
16042            assert_eq!(
16043                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16044                    .await
16045                    .unwrap(),
16046                1
16047            );
16048            runner.state = RunState::load(&runner.state.id).unwrap();
16049        }
16050    }
16051
16052    #[tokio::test]
16053    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16054        let (_tmp, mut runner) = preservation_resume_fixture().await;
16055        let wt = runner.state.candidates[0].worktree.clone();
16056        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16057        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16058        let index_lock = PathBuf::from(
16059            git::git(
16060                &wt,
16061                &[
16062                    "rev-parse",
16063                    "--path-format=absolute",
16064                    "--git-path",
16065                    "index.lock",
16066                ],
16067            )
16068            .await
16069            .unwrap(),
16070        );
16071        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16072        runner.state.schema = 14;
16073        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16074        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16075        assert!(runner.implement().await.is_err());
16076        assert!(runner.after_implement().is_err());
16077        assert!(runner.fold_losers().await.is_err());
16078        assert_eq!(
16079            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16080            "failed seat edits"
16081        );
16082        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16083        runner.state = RunState::load(&runner.state.id).unwrap();
16084        std::fs::remove_file(index_lock).unwrap();
16085        // Recovery finds changes that the old uncontested result never judged.
16086        assert!(runner.implement().await.is_err());
16087        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16088        assert!(runner.state.candidates[0].empty);
16089        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16090        assert!(
16091            crate::run::read_artifact(&runner.state, "cand-A.patch")
16092                .unwrap()
16093                .contains("failed seat edits")
16094        );
16095        assert!(wt.exists());
16096    }
16097
16098    #[tokio::test]
16099    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16100        let (_tmp, mut runner) = preservation_resume_fixture().await;
16101        let repo = runner.state.repo.clone();
16102        let common = git::git(
16103            &repo,
16104            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16105        )
16106        .await
16107        .unwrap();
16108        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16109        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16110        assert!(runner.fold_losers().await.is_err());
16111        assert!(!runner.state.candidates[0].worktree.exists());
16112        assert!(!runner.state.candidates[0].folded);
16113        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16114        runner.state = RunState::load(&runner.state.id).unwrap();
16115        std::fs::remove_file(branch_lock).unwrap();
16116        let judge = runner.state.worktree_root().join("judge-1");
16117        git::worktree_add_detached(&repo, &judge, "main")
16118            .await
16119            .unwrap();
16120        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16121        runner.roles.judges = vec![spec("judge")];
16122        assert!(runner.fold_losers().await.is_err());
16123        runner.state = RunState::load(&runner.state.id).unwrap();
16124        assert!(
16125            runner.state.candidates[0].folded,
16126            "candidate progress saved before judge failure"
16127        );
16128        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16129        assert!(judge.join("uncommitted").exists());
16130        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16131        runner.fold_losers().await.unwrap();
16132        assert!(!judge.exists());
16133        runner.fold_losers().await.unwrap();
16134    }
16135
16136    #[tokio::test]
16137    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16138        let (_tmp, mut runner) = preservation_resume_fixture().await;
16139        runner.state.candidates.retain(|c| c.label == 'B');
16140        runner.state.tally = None;
16141        runner.state.judge_skipped = false;
16142        runner.state.prep_complete = Some(true);
16143        runner.state.config.graph.advise = false;
16144        runner.state.config.graph.selected_implementer = Some("alpha".into());
16145        let wt = runner.state.candidates[0].worktree.clone();
16146        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16147        runner
16148            .state
16149            .candidate_preservation
16150            .insert('B', "awaiting persistence".into());
16151        let index_lock = git::git(
16152            &wt,
16153            &[
16154                "rev-parse",
16155                "--path-format=absolute",
16156                "--git-path",
16157                "index.lock",
16158            ],
16159        )
16160        .await
16161        .unwrap();
16162        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16163        runner.state.save().unwrap();
16164        assert!(runner.execute_graph().await.is_err());
16165        assert_eq!(runner.state.status, RunStatus::Blocked);
16166        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16167        std::fs::remove_file(index_lock).unwrap();
16168        // Park at the next graph boundary: this tests the real selected-seat
16169        // hold guard without dispatching reviewers or publishing anything.
16170        for _ in 0..2 {
16171            runner.state = RunState::load(&runner.state.id).unwrap();
16172            runner.pause.park();
16173            runner.execute_graph().await.unwrap();
16174            assert_eq!(runner.state.status, RunStatus::Implementing);
16175            assert!(runner.state.candidate_preservation.is_empty());
16176            assert_eq!(runner.state.candidates[0].commits, 2);
16177            assert!(runner.state.tally.is_none());
16178        }
16179        runner.implement().await.unwrap();
16180        assert_eq!(runner.state.status, RunStatus::Judging);
16181        // A genuine selected-implementer hold must still require release.
16182        runner.state.status = RunStatus::Blocked;
16183        runner.state.candidates[0].failed = Some("implementation failed".into());
16184        runner.state.save().unwrap();
16185        let error = runner.execute_graph().await.unwrap_err();
16186        assert!(
16187            error
16188                .to_string()
16189                .contains("selected implementation is held")
16190        );
16191    }
16192
16193    #[tokio::test]
16194    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16195        let (_tmp, mut runner) = preservation_resume_fixture().await;
16196        let repo = runner.state.repo.clone();
16197        let candidate = runner.state.candidates[0].clone();
16198        // Remove only the disposable fixture directory, leaving Git's linked
16199        // worktree registration and branch intact, as in a vanished checkout.
16200        let registered = git::worktree_holding(&repo, &candidate.branch)
16201            .await
16202            .unwrap()
16203            .unwrap();
16204        assert_eq!(
16205            registered.canonicalize().unwrap(),
16206            candidate.worktree.canonicalize().unwrap()
16207        );
16208        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16209        assert_eq!(
16210            git::worktree_holding(&repo, &candidate.branch)
16211                .await
16212                .unwrap(),
16213            Some(registered)
16214        );
16215        assert!(
16216            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16217            "the stale registration initially keeps the branch checked out"
16218        );
16219        runner.fold_losers().await.unwrap();
16220        assert!(runner.state.candidates[0].folded);
16221        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16222        assert_eq!(
16223            git::worktree_holding(&repo, &candidate.branch)
16224                .await
16225                .unwrap(),
16226            None
16227        );
16228        runner.state = RunState::load(&runner.state.id).unwrap();
16229        runner.fold_losers().await.unwrap();
16230    }
16231
16232    #[tokio::test]
16233    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16234        let (_tmp, mut runner) = preservation_resume_fixture().await;
16235        let repo = runner.state.repo.clone();
16236        let candidate = runner.state.candidates[0].clone();
16237        let winner = runner.state.candidates[1].clone();
16238        let registered = git::worktree_holding(&repo, &candidate.branch)
16239            .await
16240            .unwrap()
16241            .unwrap();
16242        assert_eq!(
16243            registered.canonicalize().unwrap(),
16244            candidate.worktree.canonicalize().unwrap()
16245        );
16246        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16247        assert_eq!(
16248            git::worktree_holding(&repo, &candidate.branch)
16249                .await
16250                .unwrap(),
16251            Some(registered)
16252        );
16253        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16254
16255        for _ in 0..2 {
16256            fold_run(&mut runner.state, false, &crate::run::home())
16257                .await
16258                .unwrap();
16259            runner.state = RunState::load(&runner.state.id).unwrap();
16260            assert!(runner.state.candidates[0].folded);
16261            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16262            assert_eq!(
16263                git::worktree_holding(&repo, &candidate.branch)
16264                    .await
16265                    .unwrap(),
16266                None
16267            );
16268            assert!(!runner.state.candidates[1].folded);
16269            assert!(winner.worktree.exists());
16270            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16271        }
16272    }
16273
16274    #[tokio::test]
16275    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16276        ask_test_home();
16277        let tmp = tempfile::tempdir().unwrap();
16278        let repo = tmp.path().join("repo");
16279        std::fs::create_dir(&repo).unwrap();
16280        init_repo(&repo);
16281        let mut runner = runner_at(RunStatus::Implementing);
16282        runner.state.repo = repo.clone();
16283        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16284        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16285        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16286            let branch = format!("candidate-{label}");
16287            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16288            git::worktree_add_branch(&repo, &wt, &branch, "main")
16289                .await
16290                .unwrap();
16291            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16292            git::git(&wt, &["add", "README.md"]).await.unwrap();
16293            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16294            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16295            runner.state.candidates.push(Candidate {
16296                index,
16297                label,
16298                agent: "alpha".to_owned(),
16299                branch,
16300                worktree: wt,
16301                summary: "completed implementation".to_owned(),
16302                stat: String::new(),
16303                files: 0,
16304                commits: 0,
16305                empty: false,
16306                failed: None,
16307                verified_noop: None,
16308                duration_ms: 42,
16309                folded: false,
16310            });
16311            runner
16312                .state
16313                .candidate_preservation
16314                .insert(label, "awaiting persistence".to_owned());
16315        }
16316        let wt = runner.state.candidates[0].worktree.clone();
16317        let index_lock = git::git(
16318            &wt,
16319            &[
16320                "rev-parse",
16321                "--path-format=absolute",
16322                "--git-path",
16323                "index.lock",
16324            ],
16325        )
16326        .await
16327        .unwrap();
16328        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16329        runner.state.save().unwrap();
16330        assert!(runner.persist_candidates().await.is_err());
16331        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16332        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16333        assert_eq!(
16334            runner.state.candidates[1].commits, 1,
16335            "other results are still collected"
16336        );
16337        assert!(
16338            !runner.state.candidates[0].empty,
16339            "failed rescue is not no change"
16340        );
16341        assert_eq!(
16342            runner.state.candidates[0].summary,
16343            "completed implementation"
16344        );
16345        assert!(runner.after_implement().is_err());
16346        let home = tmp.path().join("fold-home");
16347        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16348        assert!(wt.join("new-file").exists());
16349        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16350        assert!(recovery.join("staged.patch").exists());
16351        assert!(recovery.join("unstaged.patch").exists());
16352        assert_eq!(
16353            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16354            "untracked edits\n"
16355        );
16356        runner.state.begin_pass();
16357        runner.state.end_pass_parked(ParkCause::Upgrade {
16358            from: "1.0".into(),
16359            to: Some("1.1".into()),
16360        });
16361        runner.state.save().unwrap();
16362        runner.state = RunState::load(&runner.state.id).unwrap();
16363        assert!(matches!(
16364            runner.state.passes.last().unwrap().end,
16365            Some(crate::run::PassEnd::Parked {
16366                cause: ParkCause::Upgrade { .. }
16367            })
16368        ));
16369        std::fs::remove_file(index_lock).unwrap();
16370        // No implementer is configured: a persistence retry cannot dispatch a seat.
16371        runner.implement().await.unwrap();
16372        assert!(runner.state.candidate_preservation.is_empty());
16373        assert!(
16374            runner
16375                .state
16376                .candidates
16377                .iter()
16378                .all(|c| c.commits == 1 && !c.empty)
16379        );
16380        assert!(git::is_clean(&wt).await.unwrap());
16381        assert!(
16382            crate::run::read_artifact(&runner.state, "cand-A.patch")
16383                .unwrap()
16384                .contains("later tracked edits")
16385        );
16386    }
16387
16388    #[tokio::test]
16389    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16390        ask_test_home();
16391        let tmp = tempfile::tempdir().unwrap();
16392        let repo = tmp.path().join("repo");
16393        std::fs::create_dir(&repo).unwrap();
16394        init_repo(&repo);
16395        let wt = tmp.path().join("candidate");
16396        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16397            .await
16398            .unwrap();
16399        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16400        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16401        assert!(wt.join("uncommitted").exists());
16402        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16403        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16404        let mut runner = runner_at(RunStatus::Implementing);
16405        runner.state.repo = repo.clone();
16406        runner.state.base_commit = "missing-base".to_owned();
16407        runner.state.candidates.push(Candidate {
16408            index: 0,
16409            label: 'A',
16410            agent: "alpha".into(),
16411            branch: "candidate".into(),
16412            worktree: wt.clone(),
16413            summary: "completed".into(),
16414            stat: String::new(),
16415            files: 0,
16416            commits: 0,
16417            empty: false,
16418            failed: None,
16419            verified_noop: None,
16420            duration_ms: 1,
16421            folded: false,
16422        });
16423        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16424        assert!(
16425            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16426                .await
16427                .is_err()
16428        );
16429        assert!(!runner.state.candidates[0].folded);
16430        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16431        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16432        runner
16433            .state
16434            .candidate_preservation
16435            .insert('A', "pending".into());
16436        assert!(runner.persist_candidates().await.is_err());
16437        assert!(!runner.state.candidates[0].empty);
16438        assert!(!runner.state.candidates[0].folded);
16439        assert!(wt.exists());
16440        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16441        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16442        std::fs::create_dir_all(&patch_path).unwrap();
16443        assert!(
16444            runner.persist_candidates().await.is_err(),
16445            "artifact failure stays pending"
16446        );
16447        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16448        assert!(!runner.state.candidates[0].empty);
16449        std::fs::remove_dir(patch_path).unwrap();
16450        runner.persist_candidates().await.unwrap();
16451        assert!(
16452            runner.state.candidates[0].empty,
16453            "only successful inspection confirms no change"
16454        );
16455    }
16456
16457    fn init_repo(dir: &Path) {
16458        let run = |args: &[&str]| {
16459            let out = std::process::Command::new("git")
16460                .args(args)
16461                .current_dir(dir)
16462                .quiet()
16463                .output()
16464                .expect("spawn git");
16465            assert!(
16466                out.status.success(),
16467                "git {args:?} failed: {}",
16468                String::from_utf8_lossy(&out.stderr)
16469            );
16470        };
16471        run(&["init", "-b", "main"]);
16472        run(&["config", "user.name", "magi test"]);
16473        run(&["config", "user.email", "magi@example.com"]);
16474        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16475        run(&["add", "-A"]);
16476        run(&["commit", "-m", "init"]);
16477    }
16478
16479    // `settle_questions` is what closes the ghost the phone showed: a run's
16480    // seat asked something, the run then ended, and nothing was left to
16481    // abandon the question it left `open`. `HOME` is a process-wide
16482    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16483    // first time it runs in the binary — every test below still reaches the
16484    // same directory whichever call won, and each gets its own run id from
16485    // `RunState::new`, so they never collide there.
16486    fn ask_test_home() {
16487        crate::run::pin_test_home();
16488    }
16489
16490    #[test]
16491    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16492        ask_test_home();
16493        let mut runner = runner_at(RunStatus::Prep);
16494        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16495        // A blocked temporary output path makes the terminal save fail while
16496        // preserving the already published reservation.
16497        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16498        let error = guard
16499            .finish(
16500                &mut runner.state,
16501                Err(
16502                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16503                        .into(),
16504                ),
16505            )
16506            .unwrap_err();
16507        assert_eq!(
16508            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16509            std::io::ErrorKind::PermissionDenied
16510        );
16511        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16512        let message = format!("{error:#}");
16513        assert!(message.contains("original refusal"));
16514        assert!(!message.contains("could not save terminal"));
16515        assert_eq!(
16516            error.chain().count(),
16517            2,
16518            "only startup attribution wraps the original error"
16519        );
16520        let saved = RunState::load(&runner.state.id).unwrap();
16521        assert_eq!(saved.status, RunStatus::Blocked);
16522        assert!(!saved.startup.unwrap().complete);
16523    }
16524
16525    #[test]
16526    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16527        ask_test_home();
16528        let mut runner = runner_at(RunStatus::Prep);
16529        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16530        let id = runner.state.id.clone();
16531        drop(guard);
16532        let saved = RunState::load(&id).unwrap();
16533        assert_eq!(saved.status, RunStatus::Blocked);
16534        assert!(!saved.startup.as_ref().unwrap().complete);
16535        assert!(
16536            saved
16537                .startup
16538                .unwrap()
16539                .failure
16540                .unwrap()
16541                .contains("interrupted")
16542        );
16543        assert!(Runner::resume(&id).is_err());
16544    }
16545
16546    #[test]
16547    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16548        ask_test_home();
16549        let mut runner = runner_at(RunStatus::Prep);
16550        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16551        let error = guard
16552            .finish(
16553                &mut runner.state,
16554                Err(
16555                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16556                        .into(),
16557                ),
16558            )
16559            .unwrap_err();
16560        assert_eq!(
16561            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16562            std::io::ErrorKind::PermissionDenied
16563        );
16564        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16565        let mut task = crate::queue::Task::new(
16566            "review".into(),
16567            "criteria".into(),
16568            runner.state.repo.clone(),
16569            crate::queue::Source::Human,
16570        );
16571        task.record_startup_failure(failure.clone());
16572        task.record_startup_failure(failure.clone());
16573        assert_eq!(task.startup_failures.len(), 1);
16574        assert!(task.runs.is_empty());
16575        assert_eq!(task.attempts, 0);
16576    }
16577
16578    #[cfg(unix)]
16579    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16580        ask_test_home();
16581        let tmp = tempfile::tempdir().unwrap();
16582        let repo = tmp.path().join("repo");
16583        std::fs::create_dir(&repo).unwrap();
16584        init_repo(&repo);
16585        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16586        let origin = tmp.path().join("origin.git");
16587        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16588            .await
16589            .unwrap();
16590        git::git(
16591            &repo,
16592            &["remote", "add", "origin", origin.to_str().unwrap()],
16593        )
16594        .await
16595        .unwrap();
16596        git::git(&repo, &["checkout", "-b", "fix/retained"])
16597            .await
16598            .unwrap();
16599        std::fs::write(repo.join("retained"), "retained change").unwrap();
16600        git::git(&repo, &["add", "."]).await.unwrap();
16601        git::git(&repo, &["commit", "-m", "retained implementation"])
16602            .await
16603            .unwrap();
16604        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16605        git::git(&repo, &["checkout", "main"]).await.unwrap();
16606        std::fs::write(repo.join("base-change"), "new base").unwrap();
16607        git::git(&repo, &["add", "."]).await.unwrap();
16608        git::git(&repo, &["commit", "-m", "base moved"])
16609            .await
16610            .unwrap();
16611        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16612        git::git(&repo, &["checkout", "fix/retained"])
16613            .await
16614            .unwrap();
16615        let mut runner = runner_at(RunStatus::Voting);
16616        runner.state.repo = repo.clone();
16617        runner.state.base_commit = base;
16618        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16619        runner.state.config.disk.min_free_bytes = 0;
16620        runner.state.config.graph.review_rounds = 1;
16621        runner.state.config.verify.e2e = vec!["true".into()];
16622        runner.state.config.verify.gate = vec!["true".into()];
16623        runner.state.candidates = vec![Candidate {
16624            index: 0,
16625            label: 'A',
16626            agent: "fixture".into(),
16627            branch: "fix/retained".into(),
16628            worktree: repo.clone(),
16629            summary: String::new(),
16630            stat: String::new(),
16631            files: 1,
16632            commits: 1,
16633            empty: false,
16634            failed: None,
16635            verified_noop: None,
16636            duration_ms: 0,
16637            folded: false,
16638        }];
16639        runner.state.tally = Some(Tally {
16640            first_choice: BTreeMap::new(),
16641            borda: BTreeMap::new(),
16642            winner: 'A',
16643            rankings: 0,
16644            unanimous_initial: true,
16645            deliberated: false,
16646            changed_votes: 0,
16647            unanimous_final: true,
16648            tie_break: None,
16649            judges: 0,
16650            present: 0,
16651            quorum: 0,
16652            met_quorum: true,
16653            uncontested: Some("review-only".into()),
16654        });
16655        let mut task = crate::queue::Task::new(
16656            "fixture".into(),
16657            "fixture".into(),
16658            repo,
16659            crate::queue::Source::Human,
16660        );
16661        task.link_run(&runner.state.id);
16662        crate::queue::Queue::open().put(&mut task).unwrap();
16663        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16664        runner.state.base_sync = Some(BaseSync {
16665            tip: String::new(),
16666            behind: 1,
16667            attempts: BASE_SYNC_ROUNDS,
16668            conflict: None,
16669            already_in: None,
16670        });
16671        runner.sync_to_base().await.unwrap();
16672        assert_eq!(runner.state.status, RunStatus::Blocked);
16673        (tmp, runner, head, task.id)
16674    }
16675
16676    #[cfg(unix)]
16677    #[tokio::test]
16678    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16679        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16680        if std::env::var_os(CHILD).is_none() {
16681            let home = tempfile::tempdir().unwrap();
16682            let output = std::process::Command::new(std::env::current_exe().unwrap())
16683                .args([
16684                    "--exact",
16685                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16686                    "--nocapture",
16687                ])
16688                .env(CHILD, "1")
16689                .env("MAGI_HOME", home.path())
16690                .env("MAGI_WORKER_HOME", home.path())
16691                .env("MAGI_WORKER_ID", "abcdef")
16692                .output()
16693                .unwrap();
16694            assert!(
16695                output.status.success()
16696                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16697                "{}\n{}",
16698                String::from_utf8_lossy(&output.stdout),
16699                String::from_utf8_lossy(&output.stderr)
16700            );
16701            return;
16702        }
16703        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16704        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16705        let home = crate::run::home();
16706        let queue = crate::queue::Queue::open();
16707        let mut owner = queue.get(&task).unwrap();
16708        owner.release_operator("test-release", None);
16709        owner.start(runner.state.id.clone());
16710        queue.put(&mut owner).unwrap();
16711        runner.state.driver_exited = true;
16712        runner.state.task_instruction = Some(owner.instruction.clone());
16713        runner.state.save().unwrap();
16714        let _claim = queue.claim(&task).unwrap();
16715        let reserve = |id: &str| {
16716            let directory = home.join("workers/jobs").join(id);
16717            std::fs::create_dir_all(&directory).unwrap();
16718            let job = crate::worker::Job {
16719                id: id.into(),
16720                sequence: 1,
16721                kind: "task".into(),
16722                key: format!("task:{task}"),
16723                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16724                generation: "fixture-generation".into(),
16725                version: None,
16726                status: crate::worker::Status::Running,
16727                result: None,
16728                pid: Some(std::process::id()),
16729                created_at: 0,
16730                finished_at: None,
16731            };
16732            std::fs::write(
16733                directory.join("job.json"),
16734                serde_json::to_vec(&job).unwrap(),
16735            )
16736            .unwrap();
16737            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16738        };
16739        let _own = reserve("abcdef");
16740        let other = reserve("fedcba");
16741        assert!(
16742            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16743                .await
16744                .is_err()
16745        );
16746        assert!(
16747            RunState::load(&runner.state.id)
16748                .unwrap()
16749                .base_sync_recovery
16750                .authorizations
16751                .is_empty()
16752        );
16753        drop(other);
16754        assert!(crate::daemon::is_working_on(
16755            &home,
16756            &runner.state.id,
16757            Timestamp::now()
16758        ));
16759        assert!(
16760            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16761        );
16762        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16763            .await
16764            .unwrap();
16765        assert!(
16766            RunState::load(&runner.state.id)
16767                .unwrap()
16768                .base_recovery_pending()
16769        );
16770    }
16771
16772    #[cfg(unix)]
16773    #[tokio::test]
16774    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16775        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16776        let home = crate::run::home();
16777        runner.state.driver_pid = Some(std::process::id());
16778        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16779        runner.state.driver_exited = false;
16780        runner.state.begin_pass();
16781        runner.state.save().unwrap();
16782        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16783        driver.record(&runner.state).unwrap();
16784        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16785        // Preserve the durable claim as if the process died before Drop saved.
16786        drop(driver);
16787        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16788        assert_eq!(
16789            runner.state.liveness_under(&home, false),
16790            crate::run::Liveness::Dead
16791        );
16792        let queue = crate::queue::Queue::open();
16793        let _claim = queue.claim(&task).unwrap();
16794        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16795            .await
16796            .unwrap();
16797        let disk = RunState::load(&runner.state.id).unwrap();
16798        assert!(!disk.driver_exited);
16799        assert!(disk.base_recovery_pending());
16800    }
16801
16802    #[cfg(unix)]
16803    #[tokio::test]
16804    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16805        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16806        if std::env::var_os(CHILD).is_none() {
16807            use std::os::unix::fs::PermissionsExt;
16808            let tmp = tempfile::tempdir().unwrap();
16809            let gh = tmp.path().join("gh");
16810            std::fs::write(&gh, r#"#!/bin/sh
16811case "$1 $2" in
16812  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16813  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16814  *) exit 2 ;;
16815esac
16816"#).unwrap();
16817            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16818            let mut paths = vec![tmp.path().to_path_buf()];
16819            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16820            let output = std::process::Command::new(std::env::current_exe().unwrap())
16821                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16822                .env(CHILD, "1")
16823                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16824                .env("PATH", std::env::join_paths(paths).unwrap())
16825                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16826                .output().unwrap();
16827            assert!(
16828                output.status.success()
16829                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16830                "{}\n{}",
16831                String::from_utf8_lossy(&output.stdout),
16832                String::from_utf8_lossy(&output.stderr)
16833            );
16834            return;
16835        }
16836        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16837        let repo = runner.state.repo.clone();
16838        git::git(&repo, &["checkout", "main"]).await.unwrap();
16839        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16840        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16841        git::git(&repo, &["checkout", "fix/retained"])
16842            .await
16843            .unwrap();
16844        runner.state.pr = Some(crate::run::PrRecord {
16845            url: "https://example.test/project/pull/1".into(),
16846            number: 1,
16847            state: "open".into(),
16848            checks: "unknown".into(),
16849            round: 0,
16850            rounds: 1,
16851            red_at_merge: Vec::new(),
16852        });
16853        runner.sync_to_base().await.unwrap();
16854        assert_eq!(runner.state.status, RunStatus::Blocked);
16855        assert_eq!(
16856            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16857            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16858        );
16859        assert!(runner.state.base_sync_closure_pending());
16860        assert!(crate::handover::can_resume(&runner.state));
16861        let queue = crate::queue::Queue::open();
16862        assert!(!crate::base_sync_recovery::held_task(
16863            &queue,
16864            &queue.get(&task).unwrap()
16865        ));
16866        assert!(runner.state.authorize_base_sync(&task).is_err());
16867        // Legacy closure stops have no typed stop but must retain the checkpoint.
16868        runner.state.base_sync_recovery.stop = None;
16869        runner.state.status = RunStatus::Voting;
16870        runner.state.gate_ran = true;
16871        runner.state.config.graph.review_rounds = 0;
16872        runner.state.save().unwrap();
16873        assert!(crate::handover::can_resume(&runner.state));
16874        runner.state = RunState::load(&runner.state.id).unwrap();
16875        runner.execute_graph().await.unwrap();
16876        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16877        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16878        assert_eq!(
16879            runner.state.base_sync.as_ref().unwrap().attempts,
16880            BASE_SYNC_ROUNDS
16881        );
16882        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16883        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16884        assert!(runner.state.reviews.is_empty());
16885        assert!(runner.state.gate.is_empty());
16886    }
16887
16888    #[cfg(unix)]
16889    #[tokio::test]
16890    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16891        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16892        runner.state.driver_exited = true;
16893        runner.state.save().unwrap();
16894        let home = crate::run::home();
16895        let queue = crate::queue::Queue::open();
16896        let _claim = queue.claim(&task).unwrap();
16897        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16898        assert!(
16899            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16900                .await
16901                .is_err()
16902        );
16903        drop(lease);
16904        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16905        assert!(
16906            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16907                .await
16908                .is_err()
16909        );
16910        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16911        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16912            .await
16913            .unwrap();
16914        let disk = RunState::load(&runner.state.id).unwrap();
16915        assert!(disk.base_recovery_pending());
16916        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16917        assert!(
16918            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16919                .await
16920                .is_err()
16921        );
16922        assert_eq!(
16923            RunState::load(&runner.state.id)
16924                .unwrap()
16925                .base_sync_recovery
16926                .authorizations
16927                .len(),
16928            1
16929        );
16930        let mut owner = queue.get(&task).unwrap();
16931        owner.runs.push("20260101-000000-ffff".into());
16932        queue.put(&mut owner).unwrap();
16933        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16934            .await
16935            .unwrap_err();
16936        assert!(error.to_string().contains("run task ownership changed"));
16937    }
16938
16939    #[cfg(unix)]
16940    #[tokio::test]
16941    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16942        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16943        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16944        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16945        legacy["status"] = serde_json::json!("voting");
16946        runner.state = serde_json::from_value(legacy).unwrap();
16947        runner.state.save().unwrap();
16948        for _ in 0..3 {
16949            runner.execute_graph().await.unwrap();
16950            assert_eq!(runner.state.status, RunStatus::Blocked);
16951            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16952            assert!(!crate::handover::can_resume(&runner.state));
16953            assert_eq!(
16954                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16955                head
16956            );
16957            assert!(runner.state.reviews.is_empty());
16958            assert!(!runner.state.gate_ran);
16959        }
16960        let notice = crate::notices::run_ended(&runner.state).unwrap();
16961        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16962        runner.state.begin_pass();
16963        runner.state.status = RunStatus::Voting;
16964        runner.deliberate().await.unwrap();
16965        runner.vote().await.unwrap();
16966        runner.finish_after_tally().await.unwrap();
16967        assert_eq!(runner.state.status, RunStatus::Blocked);
16968        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16969        assert_eq!(
16970            crate::notices::run_ended(&runner.state).unwrap().since,
16971            notice.since
16972        );
16973    }
16974
16975    #[cfg(unix)]
16976    #[tokio::test]
16977    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16978        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16979        for detail in [
16980            "fix/retained conflicts with origin/main and did not rebase: conflict",
16981            "rebased fix/retained locally but origin refused the push",
16982            "unknown failure",
16983        ] {
16984            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16985            runner.state.record_base_sync_stop(&head);
16986            assert!(runner.state.authorize_base_sync(&task).is_err());
16987            runner.state.status = RunStatus::Voting;
16988            runner.execute_graph().await.unwrap();
16989            assert_eq!(runner.state.status, RunStatus::Blocked);
16990            assert!(!crate::handover::can_resume(&runner.state));
16991            assert_eq!(
16992                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16993                head
16994            );
16995        }
16996    }
16997
16998    #[cfg(unix)]
16999    #[tokio::test]
17000    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17001        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17002        runner.state.authorize_base_sync(&task).unwrap();
17003        assert!(runner.state.consume_base_recovery(&head));
17004        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17005        runner.state.reserve_base_operation(&head, &tip);
17006        runner.state.save().unwrap();
17007        runner.state = RunState::load(&runner.state.id).unwrap();
17008        assert!(!runner.state.consume_base_recovery(&head));
17009        runner.sync_to_base().await.unwrap();
17010        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17011        assert_eq!(runner.state.base_episode_used(), 1);
17012        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17013        assert!(
17014            runner
17015                .state
17016                .base_sync_recovery
17017                .operations
17018                .last()
17019                .unwrap()
17020                .finished
17021        );
17022        assert!(!runner.state.base_sync_blocked());
17023        runner.sync_to_base().await.unwrap();
17024        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17025    }
17026
17027    #[cfg(unix)]
17028    #[tokio::test]
17029    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17030        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17031        // Both historical diagnostic spellings retain an exhaustion identity.
17032        let original = runner
17033            .state
17034            .base_sync
17035            .as_ref()
17036            .unwrap()
17037            .conflict
17038            .clone()
17039            .unwrap();
17040        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17041            original
17042                .replace("synchronization(s)", "rebase(s)")
17043                .replace("synchronizing", "rebasing"),
17044        );
17045        runner.state.base_sync_recovery = Default::default();
17046        runner.state.record_base_sync_stop(&head);
17047        assert_eq!(
17048            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17049            crate::base_sync_recovery::StopKind::Exhausted
17050        );
17051        runner.state.authorize_base_sync(&task).unwrap();
17052        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17053        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17054            .as_object_mut()
17055            .unwrap();
17056        auth.remove("interval");
17057        auth.remove("scope");
17058        runner.state = serde_json::from_value(legacy).unwrap();
17059        assert!(runner.state.consume_base_recovery(&head));
17060        assert_eq!(runner.state.base_episode_used(), 3);
17061        runner.sync_to_base().await.unwrap();
17062        assert_eq!(runner.state.base_episode_used(), 4);
17063        runner.state.resource_block = Some(crate::run::ResourceBlock {
17064            node: "gate".into(),
17065            kind: "cache".into(),
17066            detail: "legacy wait".into(),
17067            at: Timestamp::now(),
17068            cause: crate::run::ResourceBlockCause::Unrecorded,
17069        });
17070        runner.state.save().unwrap();
17071        assert!(
17072            runner
17073                .state
17074                .base_sync_recovery
17075                .episodes
17076                .last()
17077                .unwrap()
17078                .boundary
17079                .is_none()
17080        );
17081        runner.state.resource_block = None;
17082        advance_base_fixture(&runner).await;
17083        runner.sync_to_base().await.unwrap();
17084        assert!(runner.state.base_sync_blocked());
17085        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17086    }
17087
17088    #[cfg(unix)]
17089    async fn advance_base_fixture(runner: &Runner) {
17090        let repo = &runner.state.repo;
17091        for args in [
17092            vec!["checkout", "main"],
17093            vec!["commit", "--allow-empty", "-m", "base advanced"],
17094            vec!["push", "origin", "main"],
17095            vec!["checkout", "fix/retained"],
17096        ] {
17097            git::git(repo, &args).await.unwrap();
17098        }
17099    }
17100
17101    #[cfg(unix)]
17102    fn fresh_base_interval_fixture(runner: &mut Runner) {
17103        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17104        runner.state.base_sync.as_mut().unwrap().conflict = None;
17105        runner.state.base_sync_recovery = Default::default();
17106        runner.state.status = RunStatus::Voting;
17107    }
17108
17109    #[cfg(unix)]
17110    #[tokio::test]
17111    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17112        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17113        fresh_base_interval_fixture(&mut runner);
17114        runner.state.begin_pass();
17115        runner.sync_to_base().await.unwrap();
17116        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17117        runner.state.resource_block = Some(crate::run::ResourceBlock {
17118            node: "gate".into(),
17119            kind: "build cache".into(),
17120            detail: "busy".into(),
17121            at: Timestamp::now(),
17122            cause: crate::run::ResourceBlockCause::Timeout {
17123                waited_secs: 1,
17124                limit_secs: 1,
17125            },
17126        });
17127        runner.state.save().unwrap();
17128        let queue = crate::queue::Queue::open();
17129        let mut owner = queue.get(&task).unwrap();
17130        runner.state.task_instruction = Some(owner.instruction.clone());
17131        owner.release_operator("fixture-owner", None);
17132        queue.put(&mut owner).unwrap();
17133        let approval = owner.resumption.clone().unwrap();
17134        runner.state.resource_block = None;
17135        runner.state.save().unwrap();
17136        runner.state = RunState::load(&runner.state.id).unwrap();
17137        advance_base_fixture(&runner).await;
17138        runner.sync_to_base().await.unwrap();
17139        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17140        assert_eq!(runner.state.base_episode_used(), 1);
17141        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17142        assert!(
17143            runner.state.base_sync_recovery.episodes[0]
17144                .boundary
17145                .as_ref()
17146                .unwrap()
17147                .consumed
17148        );
17149        assert!(
17150            !runner
17151                .state
17152                .renew_base_episode(&task, &approval, &head, "base")
17153        );
17154        for _ in 0..3 {
17155            owner.release_operator("fixture-owner", None);
17156            queue.put(&mut owner).unwrap();
17157            runner.state = RunState::load(&runner.state.id).unwrap();
17158            runner.state.begin_pass();
17159            runner.sync_to_base().await.unwrap();
17160            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17161            assert_eq!(runner.state.base_episode_used(), 1);
17162        }
17163        runner.state.resource_block = Some(crate::run::ResourceBlock {
17164            node: "gate".into(),
17165            kind: "disk pressure".into(),
17166            detail: "free space required".into(),
17167            at: Timestamp::now(),
17168            cause: crate::run::ResourceBlockCause::Other,
17169        });
17170        runner.state.end_pass_parked(ParkCause::Interrupt {
17171            reason: "disk pressure".into(),
17172        });
17173        runner.state.save().unwrap();
17174        owner.release_operator("fixture-owner-after-disk", None);
17175        queue.put(&mut owner).unwrap();
17176        runner.state.resource_block = None;
17177        advance_base_fixture(&runner).await;
17178        runner.sync_to_base().await.unwrap();
17179        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17180        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17181        assert_eq!(runner.state.base_episode_used(), 1);
17182        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17183    }
17184
17185    #[cfg(unix)]
17186    #[tokio::test]
17187    async fn base_recovery_continuous_advancement_stops_at_four() {
17188        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17189        fresh_base_interval_fixture(&mut runner);
17190        for spent in 1..=BASE_SYNC_ROUNDS {
17191            runner.sync_to_base().await.unwrap();
17192            assert!(!runner.state.base_sync_blocked());
17193            assert_eq!(runner.state.base_episode_used(), spent);
17194            advance_base_fixture(&runner).await;
17195        }
17196        runner.sync_to_base().await.unwrap();
17197        assert_eq!(
17198            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17199            crate::base_sync_recovery::StopKind::Exhausted
17200        );
17201        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17202        assert!(stop.unverified_sync);
17203        let action = runner.state.base_sync_next_action();
17204        assert!(action.contains("never reached its gate"));
17205        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17206        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17207        runner.state = RunState::load(&runner.state.id).unwrap();
17208        runner.sync_to_base().await.unwrap();
17209        assert_eq!(runner.state.base_episode_used(), 4);
17210        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17211    }
17212
17213    #[cfg(unix)]
17214    #[tokio::test]
17215    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17216        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17217        runner.state.ensure_base_episode();
17218        // Verification progress alone is not evidence of a successful sync.
17219        runner.state.record_base_progress(&head);
17220        runner.state.base_sync_recovery.stop = None;
17221        runner.state.record_base_sync_stop(&head);
17222        assert!(
17223            !runner
17224                .state
17225                .base_sync_recovery
17226                .stop
17227                .as_ref()
17228                .unwrap()
17229                .unverified_sync
17230        );
17231
17232        runner.state.finish_base_operation(Some(&head));
17233        // A gate for another head cannot verify this synchronization.
17234        runner.state.gate_ran = true;
17235        runner.state.gate_head = Some("older-head".into());
17236        runner.state.base_sync_recovery.stop = None;
17237        runner.state.record_base_sync_stop(&head);
17238        runner.state.save().unwrap();
17239        runner.state = RunState::load(&runner.state.id).unwrap();
17240        assert!(
17241            runner
17242                .state
17243                .base_sync_recovery
17244                .stop
17245                .as_ref()
17246                .unwrap()
17247                .unverified_sync
17248        );
17249
17250        runner.state.gate_head = Some(head.clone());
17251        runner.state.base_sync_recovery.stop = None;
17252        runner.state.record_base_sync_stop(&head);
17253        assert!(
17254            !runner
17255                .state
17256                .base_sync_recovery
17257                .stop
17258                .as_ref()
17259                .unwrap()
17260                .unverified_sync
17261        );
17262        assert!(
17263            !runner
17264                .state
17265                .base_sync_next_action()
17266                .contains("never reached its gate")
17267        );
17268
17269        // Historical records cannot establish the missing synchronization evidence.
17270        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17271        legacy["base_sync_recovery"]["stop"]
17272            .as_object_mut()
17273            .unwrap()
17274            .remove("unverified_sync");
17275        legacy["base_sync_recovery"]["episodes"][0]
17276            .as_object_mut()
17277            .unwrap()
17278            .remove("last_sync_head");
17279        runner.state = serde_json::from_value(legacy).unwrap();
17280        assert!(
17281            !runner
17282                .state
17283                .base_sync_recovery
17284                .stop
17285                .as_ref()
17286                .unwrap()
17287                .unverified_sync
17288        );
17289        assert!(
17290            runner.state.base_sync_recovery.episodes[0]
17291                .last_sync_head
17292                .is_none()
17293        );
17294    }
17295
17296    #[cfg(unix)]
17297    #[tokio::test]
17298    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17299        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17300        runner.state.authorize_base_sync(&task).unwrap();
17301        assert!(
17302            runner
17303                .state
17304                .base_sync_next_action()
17305                .contains("Approval recorded")
17306        );
17307        assert!(
17308            runner
17309                .state
17310                .base_sync_next_action()
17311                .contains("4/4 in episode 0, 4 lifetime")
17312        );
17313        assert!(runner.state.consume_base_recovery(&head));
17314        assert!(
17315            runner
17316                .state
17317                .base_sync_next_action()
17318                .contains("0/4 in episode 1, 4 lifetime")
17319        );
17320    }
17321
17322    #[cfg(unix)]
17323    #[tokio::test]
17324    async fn base_recovery_changed_head_invalidates_without_spending() {
17325        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17326        runner.state.authorize_base_sync(&task).unwrap();
17327        git::git(
17328            &runner.state.repo,
17329            &["commit", "--allow-empty", "-m", "external head change"],
17330        )
17331        .await
17332        .unwrap();
17333        runner.sync_to_base().await.unwrap();
17334        let a = runner
17335            .state
17336            .base_sync_recovery
17337            .authorizations
17338            .last()
17339            .unwrap();
17340        assert!(a.consumed);
17341        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17342        runner.state = RunState::load(&runner.state.id).unwrap();
17343        assert!(
17344            runner
17345                .state
17346                .base_sync_next_action()
17347                .contains("Approval invalidated")
17348        );
17349        assert!(!runner.state.base_recovery_pending());
17350        assert!(runner.state.base_sync_blocked());
17351        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17352    }
17353
17354    #[cfg(unix)]
17355    #[tokio::test]
17356    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17357        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17358        runner.state.driver_exited = true;
17359        runner.state.save().unwrap();
17360        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17361            .await
17362            .unwrap();
17363        runner.state = RunState::load(&runner.state.id).unwrap();
17364        advance_base_fixture(&runner).await;
17365        runner.sync_to_base().await.unwrap();
17366        let a = runner
17367            .state
17368            .base_sync_recovery
17369            .authorizations
17370            .last()
17371            .unwrap();
17372        assert!(
17373            a.consumed
17374                && a.invalidated
17375                    .as_deref()
17376                    .unwrap()
17377                    .contains("fetched base changed")
17378        );
17379        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17380        assert_eq!(
17381            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17382            head
17383        );
17384        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17385            .await
17386            .unwrap();
17387        runner.state = RunState::load(&runner.state.id).unwrap();
17388        runner.sync_to_base().await.unwrap();
17389        assert!(!runner.state.base_sync_blocked());
17390        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17391    }
17392
17393    #[cfg(unix)]
17394    #[tokio::test]
17395    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17396        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17397        runner.state.authorize_base_sync(&task).unwrap();
17398        assert!(runner.state.consume_base_recovery(&head));
17399        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17400        runner.state.reserve_base_operation(&head, &tip);
17401        runner.state.save().unwrap();
17402        let scratch = runner.state.dir().join("base-sync");
17403        assert_eq!(
17404            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17405                .await
17406                .unwrap(),
17407            crate::rebase::Rebased::Applied
17408        );
17409        runner.state = RunState::load(&runner.state.id).unwrap();
17410        runner.sync_to_base().await.unwrap();
17411        assert!(
17412            runner
17413                .state
17414                .base_sync_recovery
17415                .operations
17416                .last()
17417                .unwrap()
17418                .finished
17419        );
17420        assert_eq!(runner.state.base_episode_used(), 1);
17421        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17422        assert!(!runner.state.base_sync_blocked());
17423    }
17424
17425    #[cfg(unix)]
17426    #[tokio::test]
17427    async fn base_recovery_crash_preserves_untracked_files() {
17428        for ignored in [false, true] {
17429            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17430            runner.state.authorize_base_sync(&task).unwrap();
17431            assert!(runner.state.consume_base_recovery(&head));
17432            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17433            runner.state.reserve_base_operation(&head, &tip);
17434            runner.state.save().unwrap();
17435            let scratch = runner.state.dir().join("base-sync");
17436            assert_eq!(
17437                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17438                    .await
17439                    .unwrap(),
17440                crate::rebase::Rebased::Applied
17441            );
17442            let repo = &runner.state.repo;
17443            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17444            assert_ne!(result, head);
17445            if ignored {
17446                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17447                    .await
17448                    .unwrap();
17449                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17450            }
17451            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17452            assert!(
17453                git::git_raw(repo, &["diff", "--quiet", &head])
17454                    .await
17455                    .unwrap()
17456                    .ok()
17457            );
17458            // Resume after the branch moved, before its checkout was reset.
17459            runner.state = RunState::load(&runner.state.id).unwrap();
17460            runner.sync_to_base().await.unwrap();
17461            assert!(runner.state.base_sync_blocked());
17462            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17463            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17464            assert_eq!(
17465                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17466                result
17467            );
17468            assert_eq!(
17469                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17470                "retained work\n"
17471            );
17472            assert!(
17473                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17474                    .await
17475                    .unwrap()
17476                    .ok()
17477            );
17478        }
17479    }
17480
17481    #[cfg(unix)]
17482    #[tokio::test]
17483    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17484        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17485        runner.state.authorize_base_sync(&task).unwrap();
17486        assert!(runner.state.consume_base_recovery(&head));
17487        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17488        runner.state.reserve_base_operation(&head, &tip);
17489        runner.state.save().unwrap();
17490        let scratch = runner.state.dir().join("base-sync");
17491        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17492            .await
17493            .unwrap();
17494        git::sync_to_head(&runner.state.repo).await.unwrap();
17495        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17496        git::git(&runner.state.repo, &["add", "foreign"])
17497            .await
17498            .unwrap();
17499        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17500            .await
17501            .unwrap();
17502        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17503        runner.state = RunState::load(&runner.state.id).unwrap();
17504        runner.sync_to_base().await.unwrap();
17505        assert!(runner.state.base_sync_blocked());
17506        assert!(
17507            !runner
17508                .state
17509                .base_sync_recovery
17510                .operations
17511                .last()
17512                .unwrap()
17513                .finished
17514        );
17515        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17516        assert_eq!(
17517            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17518            foreign_head
17519        );
17520    }
17521
17522    #[cfg(unix)]
17523    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17524    {
17525        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17526        fresh_base_interval_fixture(&mut runner);
17527        let repo = &runner.state.repo;
17528        git::git(repo, &["checkout", "main"]).await.unwrap();
17529        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17530        git::git(repo, &["add", "retained"]).await.unwrap();
17531        git::git(repo, &["commit", "-m", "upstream retained change"])
17532            .await
17533            .unwrap();
17534        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17535        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17536        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17537        let scratch = runner.state.dir().join("base-sync");
17538        assert!(matches!(
17539            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17540                .await
17541                .unwrap(),
17542            git::RebaseStart::Conflicted(_)
17543        ));
17544        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17545            agent: "fixture".into(),
17546            paths: vec!["retained".into()],
17547            from: Some(head.clone()),
17548            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17549            index_outside: None,
17550            finished: false,
17551            error: None,
17552            launcher: None,
17553            infra: None,
17554            outage: None,
17555        });
17556        (tmp, runner, head, tip, scratch)
17557    }
17558
17559    #[cfg(unix)]
17560    #[tokio::test]
17561    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17562        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17563        // A resource stop from before reservations existed, with resolved
17564        // contents left for the launcher's ordinary staging continuation.
17565        std::fs::write(
17566            scratch.join("retained"),
17567            "upstream and retained resolution\n",
17568        )
17569        .unwrap();
17570        runner.state.resource_block = Some(crate::run::ResourceBlock {
17571            node: "rebase".into(),
17572            kind: "repository coordination".into(),
17573            detail: "busy".into(),
17574            at: Timestamp::now(),
17575            cause: crate::run::ResourceBlockCause::Other,
17576        });
17577        runner.state.save().unwrap();
17578        // Advance the remote without moving either checkout participating
17579        // in the standing rebase.
17580        let newer = git::git(
17581            &runner.state.repo,
17582            &[
17583                "commit-tree",
17584                "main^{tree}",
17585                "-p",
17586                "main",
17587                "-m",
17588                "base advanced during wait",
17589            ],
17590        )
17591        .await
17592        .unwrap();
17593        git::git(
17594            &runner.state.repo,
17595            &[
17596                "push",
17597                "origin",
17598                &format!("{}:refs/heads/main", newer.trim()),
17599            ],
17600        )
17601        .await
17602        .unwrap();
17603        runner.state = RunState::load(&runner.state.id).unwrap();
17604        runner.state.resource_block = None;
17605        runner.sync_to_base().await.unwrap();
17606        assert!(!runner.state.base_sync_blocked());
17607        let op = &runner.state.base_sync_recovery.operations[0];
17608        assert_eq!(op.head, head);
17609        assert_eq!(op.tip, tip);
17610        assert!(op.finished);
17611        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17612        assert_eq!(runner.state.rebase_fixes.len(), 1);
17613        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17614            .await
17615            .unwrap();
17616        assert_ne!(latest, tip);
17617        runner.state = RunState::load(&runner.state.id).unwrap();
17618        runner.sync_to_base().await.unwrap();
17619        assert!(!runner.state.base_sync_blocked());
17620        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17621        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17622        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17623    }
17624
17625    #[cfg(unix)]
17626    #[tokio::test]
17627    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17628        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17629        runner.state.reserve_base_operation(&head, &tip);
17630        runner.state.save().unwrap();
17631        std::fs::write(
17632            scratch.join("retained"),
17633            "upstream and retained resolution\n",
17634        )
17635        .unwrap();
17636        git::git(&scratch, &["add", "retained"]).await.unwrap();
17637        git::git(
17638            &scratch,
17639            &["-c", "core.editor=true", "rebase", "--continue"],
17640        )
17641        .await
17642        .unwrap();
17643        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17644        assert!(
17645            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17646                .await
17647                .unwrap()
17648                .is_empty()
17649        );
17650        assert_eq!(
17651            crate::rebase::finish(
17652                &mut runner.state,
17653                &scratch,
17654                "fix/retained",
17655                &head,
17656                &tip,
17657                &["retained".into()],
17658                ""
17659            )
17660            .await
17661            .unwrap(),
17662            crate::rebase::Rebased::Applied
17663        );
17664        assert!(!scratch.exists());
17665        // Simulate losing the caller before finish_base_operation/save.
17666        runner.state = RunState::load(&runner.state.id).unwrap();
17667        let op = &runner.state.base_sync_recovery.operations[0];
17668        assert!(!op.finished);
17669        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17670        runner.sync_to_base().await.unwrap();
17671        assert!(!runner.state.base_sync_blocked());
17672        assert!(runner.state.base_sync_recovery.operations[0].finished);
17673        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17674        assert_eq!(runner.state.rebase_fixes.len(), 1);
17675        let winner = runner.state.winner().unwrap().clone();
17676        let wait = runner.repository_wait();
17677        assert!(
17678            verify_current_head(&mut runner.state, &winner, &wait)
17679                .await
17680                .unwrap()
17681        );
17682        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17683    }
17684
17685    #[cfg(unix)]
17686    #[tokio::test]
17687    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17688        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17689            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17690            runner.state.reserve_base_operation(&head, &tip);
17691            runner.state.save().unwrap();
17692            std::fs::write(
17693                scratch.join("retained"),
17694                "upstream and retained resolution\n",
17695            )
17696            .unwrap();
17697            git::git(&scratch, &["add", "retained"]).await.unwrap();
17698            git::git(
17699                &scratch,
17700                &["-c", "core.editor=true", "rebase", "--continue"],
17701            )
17702            .await
17703            .unwrap();
17704            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17705            assert!(!git::rebase_in_progress(&scratch).await);
17706            assert!(
17707                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17708                    .await
17709                    .unwrap()
17710                    .is_empty()
17711            );
17712            match damage {
17713                Some("record") => {
17714                    let path =
17715                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17716                            .await
17717                            .unwrap();
17718                    std::fs::write(scratch.join(path), "").unwrap();
17719                }
17720                Some("foreign") => {
17721                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17722                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17723                    git::git(&scratch, &["commit", "-m", "foreign work"])
17724                        .await
17725                        .unwrap();
17726                }
17727                Some("dirty") => {
17728                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17729                }
17730                _ => {}
17731            }
17732            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17733            // Crash immediately after Git continuation, before finish runs.
17734            runner.state = RunState::load(&runner.state.id).unwrap();
17735            assert!(
17736                runner.state.base_sync_recovery.operations[0]
17737                    .validated_head
17738                    .is_none()
17739            );
17740            runner.sync_to_base().await.unwrap();
17741            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17742            assert_eq!(runner.state.base_episode_used(), 1);
17743            assert_eq!(runner.state.rebase_fixes.len(), 1);
17744            assert_eq!(
17745                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17746                retained_head
17747            );
17748            if damage.is_some() {
17749                assert!(runner.state.base_sync_blocked());
17750                assert!(scratch.exists());
17751                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17752                assert!(
17753                    runner.state.base_sync_recovery.operations[0]
17754                        .validated_head
17755                        .is_none()
17756                );
17757            } else {
17758                assert!(!runner.state.base_sync_blocked());
17759                assert!(!scratch.exists());
17760                assert!(runner.state.base_sync_recovery.operations[0].finished);
17761                assert_eq!(
17762                    runner.state.base_sync_recovery.operations[0]
17763                        .validated_head
17764                        .as_deref(),
17765                    Some(result.as_str())
17766                );
17767                let winner = runner.state.winner().unwrap().clone();
17768                let wait = runner.repository_wait();
17769                assert!(
17770                    verify_current_head(&mut runner.state, &winner, &wait)
17771                        .await
17772                        .unwrap()
17773                );
17774                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17775                runner.state = RunState::load(&runner.state.id).unwrap();
17776                runner.sync_to_base().await.unwrap();
17777                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17778            }
17779        }
17780    }
17781
17782    #[cfg(unix)]
17783    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17784        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17785        // This is an ordinary gate stop, not an authorized base-budget recovery.
17786        runner.state.prep_complete = Some(true);
17787        runner.state.base_sync = None;
17788        runner.state.base_sync_recovery = Default::default();
17789        runner.state.config.graph.gate_fix_rounds = 1;
17790        let mut review = review_round(true, 0, 1, 1, false, true);
17791        review.head = head.clone();
17792        review.verified_head = Some(head.clone());
17793        runner.state.reviews = vec![review];
17794        runner.state.gate = vec![CommandOutcome {
17795            command: "old gate".into(),
17796            code: Some(1),
17797            output_tail: "old failure".into(),
17798            duration_ms: 0,
17799            resource_blocked: false,
17800        }];
17801        runner.state.gate_head = Some(head.clone());
17802        runner.state.gate_ran = true;
17803        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17804            agent: "fixture".into(),
17805            failed: runner.state.gate.clone(),
17806            notes: "spent".into(),
17807            committed: false,
17808            error: None,
17809        }];
17810        runner.state.land_approval = Some(crate::run::LandApproval {
17811            question: "old approval".into(),
17812            head: head.clone(),
17813        });
17814        runner.state.status = RunStatus::Blocked;
17815        (tmp, runner, head)
17816    }
17817
17818    #[cfg(unix)]
17819    #[tokio::test]
17820    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17821        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17822            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17823            let phases = tmp.path().join("phases");
17824            runner.state.config.verify.e2e = vec![format!(
17825                "echo e2e >> '{}'; {}",
17826                phases.display(),
17827                if e2e_ok {
17828                    "true"
17829                } else {
17830                    "echo new-e2e-failure; false"
17831                }
17832            )];
17833            runner.state.config.verify.gate = vec![format!(
17834                "echo gate >> '{}'; {}",
17835                phases.display(),
17836                if gate_ok {
17837                    "true"
17838                } else {
17839                    "echo new-gate-failure; false"
17840                }
17841            )];
17842            runner.state.save().unwrap();
17843            runner.state = RunState::load(&runner.state.id).unwrap();
17844            runner.sync_to_base().await.unwrap();
17845            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17846            assert_ne!(head, old);
17847            // Reentry after Git updated but before verification was invalidated.
17848            runner.state = RunState::load(&runner.state.id).unwrap();
17849            runner.execute_graph().await.unwrap();
17850            assert_eq!(runner.state.gate_fixes.len(), 1);
17851            assert_eq!(runner.state.reviews.len(), 1);
17852            assert_eq!(runner.state.reviews[0].head, old);
17853            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17854            assert_eq!(runner.state.verification_history.len(), 1);
17855            let history = &runner.state.verification_history[0];
17856            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17857            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17858            assert!(history.archived_at.is_some());
17859            assert_eq!(history.fixes_spent, Some(1));
17860            assert_eq!(history.gate[0].output_tail, "old failure");
17861            let archive_events: Vec<_> = runner
17862                .state
17863                .events
17864                .iter()
17865                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17866                .collect();
17867            assert_eq!(archive_events.len(), 1);
17868            assert!(archive_events[0].message.contains(&short(&old)));
17869            assert!(archive_events[0].message.contains(&short(&head)));
17870            assert!(
17871                archive_events[0]
17872                    .message
17873                    .contains("1 gate-fix round(s) already spent stay spent")
17874            );
17875            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17876            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17877            if e2e_ok {
17878                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17879                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17880            } else {
17881                assert!(!runner.state.gate_ran);
17882            }
17883            assert_eq!(
17884                runner.state.status,
17885                if e2e_ok && gate_ok {
17886                    RunStatus::Ready
17887                } else {
17888                    RunStatus::Blocked
17889                }
17890            );
17891            if !e2e_ok || !gate_ok {
17892                let cause = runner.state.verification_blocking_cause().unwrap();
17893                assert!(cause.contains(&head));
17894                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17895                // A later successful progress event must not become the cause.
17896                runner.state.event("land", "rebased successfully");
17897                let notice = crate::notices::run_ended(&runner.state).unwrap();
17898                assert!(!format!("{notice:?}").contains("rebased successfully"));
17899                assert!(runner.state.merge.is_none());
17900            }
17901            let before = std::fs::read_to_string(&phases).unwrap();
17902            runner.state.save().unwrap();
17903            runner.state = RunState::load(&runner.state.id).unwrap();
17904            runner.review_loop().await.unwrap();
17905            runner.gate().await.unwrap();
17906            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17907            assert_eq!(runner.state.verification_history.len(), 1);
17908            let history = &runner.state.verification_history[0];
17909            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17910            assert!(history.archived_at.is_some());
17911            assert_eq!(history.fixes_spent, Some(1));
17912        }
17913    }
17914
17915    #[cfg(unix)]
17916    #[tokio::test]
17917    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17918        let (tmp, mut runner, head) = resumed_gate_fixture().await;
17919        let marker = tmp.path().join("gate-ran");
17920        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17921        runner.review_loop().await.unwrap();
17922        runner.gate().await.unwrap();
17923        assert_eq!(runner.state.status, RunStatus::Blocked);
17924        assert!(!marker.exists());
17925        assert!(runner.state.verification_history.is_empty());
17926        runner.state.gate_head = None;
17927        runner.gate().await.unwrap();
17928        assert!(marker.exists());
17929        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17930        assert_eq!(runner.state.verification_history.len(), 1);
17931        assert_eq!(runner.state.gate_fixes.len(), 1);
17932    }
17933
17934    #[cfg(unix)]
17935    #[tokio::test]
17936    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17937        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17938        runner.sync_to_base().await.unwrap();
17939        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17940        runner.state.reconcile_verification_head(&head);
17941        // Both crash-before-command and resource wait are unresolved evidence.
17942        for outcomes in [
17943            Vec::new(),
17944            vec![CommandOutcome {
17945                command: "true".into(),
17946                code: None,
17947                output_tail: "cache unavailable".into(),
17948                duration_ms: 0,
17949                resource_blocked: true,
17950            }],
17951        ] {
17952            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17953                head: head.clone(),
17954                passed: false,
17955                outcomes,
17956            });
17957            runner.state.gate_ran = false;
17958            runner.state.gate.clear();
17959            runner.state.save().unwrap();
17960            runner.state = RunState::load(&runner.state.id).unwrap();
17961            runner.merge().await.unwrap();
17962            assert!(runner.state.merge.is_none());
17963            runner.review_loop().await.unwrap();
17964            runner.gate().await.unwrap();
17965            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17966            assert!(runner.state.gate_allows_progress());
17967            assert_eq!(runner.state.gate_fixes.len(), 1);
17968            assert!(runner.state.merge.is_none());
17969        }
17970        // Moving the winner after a successful gate cannot publish an unchecked SHA.
17971        git::git(
17972            &runner.state.repo,
17973            &["commit", "--allow-empty", "-m", "external update"],
17974        )
17975        .await
17976        .unwrap();
17977        runner.merge().await.unwrap();
17978        assert!(runner.state.merge.is_none());
17979        assert_eq!(runner.state.status, RunStatus::Blocked);
17980    }
17981
17982    #[cfg(unix)]
17983    #[tokio::test]
17984    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17985        for waiting_on_gate in [false, true] {
17986            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17987            runner.sync_to_base().await.unwrap();
17988            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17989            runner.state.reconcile_verification_head(&head);
17990            if waiting_on_gate {
17991                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17992                    head: head.clone(),
17993                    passed: true,
17994                    outcomes: Vec::new(),
17995                });
17996            }
17997            let cache = tmp.path().join("cache");
17998            runner.state.config.verify.e2e =
17999                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
18000            runner.state.config.verify.gate = vec!["true".into()];
18001            runner.state.config.graph.timeout_resource_wait = 1;
18002            let owner =
18003                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18004            let guard =
18005                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18006                    crate::cache::AcquireOutcome::Acquired(g) => g,
18007                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18008                };
18009            runner.review_loop().await.unwrap();
18010            runner.gate().await.unwrap();
18011            assert!(!runner.state.gate_ran);
18012            assert_eq!(runner.state.gate_fixes.len(), 1);
18013            assert!(runner.state.resource_block.is_some());
18014            runner.merge().await.unwrap();
18015            assert!(runner.state.merge.is_none());
18016            drop(guard);
18017            runner.state = RunState::load(&runner.state.id).unwrap();
18018            // The driver clears the previous pass's resource stop on resume.
18019            runner.state.resource_block = None;
18020            runner.review_loop().await.unwrap();
18021            runner.gate().await.unwrap();
18022            assert!(runner.state.gate_allows_progress());
18023            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18024            assert_eq!(runner.state.gate_fixes.len(), 1);
18025        }
18026    }
18027
18028    #[cfg(unix)]
18029    #[tokio::test]
18030    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18031        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18032        runner.sync_to_base().await.unwrap();
18033        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18034        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18035        // Verification passes before pre_gate and fails on the mechanically changed head.
18036        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18037        runner.review_loop().await.unwrap();
18038        runner.gate().await.unwrap();
18039        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18040        assert_ne!(before, after);
18041        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18042        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18043        assert!(!runner.state.gate_allows_progress());
18044        runner.merge().await.unwrap();
18045        assert!(runner.state.merge.is_none());
18046        assert!(
18047            runner
18048                .state
18049                .verification_history
18050                .iter()
18051                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18052        );
18053
18054        // An incomplete review is still an independent hold on a changed head.
18055        runner.state.reviews[0].clean = false;
18056        runner.state.reviews[0].answered = 0;
18057        runner.state.config.verify.e2e = vec!["true".into()];
18058        runner.review_loop().await.unwrap();
18059        runner.gate().await.unwrap();
18060        runner.merge().await.unwrap();
18061        assert_eq!(runner.state.status, RunStatus::Blocked);
18062        assert!(runner.state.merge.is_none());
18063        assert!(
18064            runner
18065                .state
18066                .verification_blocking_cause()
18067                .unwrap()
18068                .contains("review blocked")
18069        );
18070    }
18071
18072    #[cfg(unix)]
18073    #[tokio::test]
18074    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18075        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18076        runner.state.gate_head = None;
18077        runner.state.config.verify.gate =
18078            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18079        runner.review_loop().await.unwrap();
18080        runner.gate().await.unwrap();
18081        assert_ne!(
18082            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18083            old
18084        );
18085        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18086        assert!(!runner.state.gate_allows_progress());
18087        assert_eq!(runner.state.status, RunStatus::Blocked);
18088        runner.merge().await.unwrap();
18089        assert!(runner.state.merge.is_none());
18090    }
18091
18092    #[cfg(unix)]
18093    #[tokio::test]
18094    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18095        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18096        let mut old_review = review_round(true, 0, 1, 1, false, true);
18097        old_review.head = head.clone();
18098        old_review.verified_head = Some(head.clone());
18099        runner.state.reviews = vec![old_review];
18100        runner.state.gate = vec![CommandOutcome {
18101            command: "old gate".into(),
18102            code: None,
18103            output_tail: "timed out after 2400s".into(),
18104            duration_ms: 2_400_000,
18105            resource_blocked: false,
18106        }];
18107        runner.state.gate_ran = true;
18108        runner.state.gate_head = Some(head.clone());
18109        runner.state.land_approval = Some(crate::run::LandApproval {
18110            question: "old-head-approval".into(),
18111            head: head.clone(),
18112        });
18113        runner.state.authorize_base_sync(&task).unwrap();
18114        runner.sync_to_base().await.unwrap();
18115        assert!(!runner.state.base_sync_blocked());
18116        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18117        assert_ne!(head, fresh);
18118        assert_eq!(runner.state.reviews.len(), 1);
18119        assert!(!runner.state.gate_ran);
18120        let saved = &runner.state.base_sync_recovery.authorizations[0];
18121        assert_eq!(saved.reviews[0].head, head);
18122        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18123        assert!(saved.gate_ran);
18124        assert!(!saved.gate[0].ok());
18125        runner.review_loop().await.unwrap();
18126        runner.gate().await.unwrap();
18127        assert_eq!(runner.state.reviews[0].head, head);
18128        assert_eq!(
18129            runner.state.reviews[0].verified_head.as_deref(),
18130            Some(head.as_str())
18131        );
18132        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18133        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18134        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18135        assert!(runner.state.gate_ran);
18136        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18137        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18138        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18139        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18140        assert_eq!(
18141            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18142            "retained change"
18143        );
18144    }
18145
18146    #[cfg(unix)]
18147    #[tokio::test]
18148    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18149        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18150        let queue = crate::queue::Queue::open();
18151        let mut owner = queue.get(&task).unwrap();
18152        owner.release_operator("test-resume", None);
18153        queue.put(&mut owner).unwrap();
18154        let generation = owner.resumption.as_ref().unwrap().generation;
18155        runner.state.task_instruction = Some(owner.instruction.clone());
18156        runner.state.driver_exited = true;
18157        runner.state.save().unwrap();
18158        let home = crate::run::home();
18159        let _claim = queue.claim(&task).unwrap();
18160        owner.resumption.as_mut().unwrap().at =
18161            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18162        queue.put(&mut owner).unwrap();
18163        assert!(
18164            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18165                .await
18166                .is_err(),
18167            "an approval from before the stop grants nothing"
18168        );
18169        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18170        queue.put(&mut owner).unwrap();
18171        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18172            .await
18173            .unwrap();
18174        assert!(
18175            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18176                .await
18177                .is_err()
18178        );
18179        runner.state = RunState::load(&runner.state.id).unwrap();
18180        assert_eq!(
18181            runner.state.base_sync_recovery.authorizations[0].generation,
18182            Some(generation)
18183        );
18184        runner.sync_to_base().await.unwrap();
18185        let repo = runner.state.repo.clone();
18186        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18187        git::git(&repo, &["checkout", "main"]).await.unwrap();
18188        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18189        git::git(&repo, &["add", "."]).await.unwrap();
18190        git::git(&repo, &["commit", "-m", "move base again"])
18191            .await
18192            .unwrap();
18193        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18194        git::git(&repo, &["checkout", "fix/retained"])
18195            .await
18196            .unwrap();
18197        runner.sync_to_base().await.unwrap();
18198        assert!(!runner.state.base_sync_blocked());
18199        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18200        assert_eq!(runner.state.base_episode_used(), 2);
18201        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18202        assert_ne!(fresh, head);
18203        assert!(!runner.state.base_recovery_pending());
18204        // Even replaying the generation with a later timestamp grants nothing.
18205        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18206        queue.put(&mut owner).unwrap();
18207        assert!(
18208            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18209                .await
18210                .is_err()
18211        );
18212        runner.state = RunState::load(&runner.state.id).unwrap();
18213        runner.sync_to_base().await.unwrap();
18214        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18215        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18216        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18217    }
18218
18219    #[cfg(unix)]
18220    #[tokio::test]
18221    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18222        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18223        runner.state.base_sync = None;
18224        runner.state.status = RunStatus::Blocked;
18225        let mut second = runner.state.candidates[0].clone();
18226        second.label = 'B';
18227        runner.state.candidates.push(second);
18228        assert!(runner.state.votes.is_empty());
18229        assert!(runner.state.deliberation.is_empty());
18230        runner.deliberate().await.unwrap();
18231        runner.vote().await.unwrap();
18232        assert_eq!(runner.state.status, RunStatus::Blocked);
18233        assert!(runner.state.votes.is_empty());
18234        assert!(runner.state.deliberation.is_empty());
18235    }
18236
18237    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18238    /// reads nothing else off it.
18239    fn runner_at(status: RunStatus) -> Runner {
18240        let mut state = RunState::new(
18241            PathBuf::from("/nonexistent/repo"),
18242            "main".to_owned(),
18243            "deadbeef".to_owned(),
18244            "task".to_owned(),
18245            Config::default(),
18246        );
18247        // These fixtures test execution flow, independent of host disk capacity.
18248        state.config.disk.min_free_bytes = 0;
18249        // These fixtures share one home and save concurrently. Short clock
18250        // IDs can collide, making an ownership test read another test's run.
18251        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18252        state.id = format!(
18253            "20000101-000000-{:04x}",
18254            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18255        );
18256        state.status = status;
18257        Runner {
18258            state,
18259            roles: ResolvedRoles {
18260                implementers: Vec::new(),
18261                judges: Vec::new(),
18262                reviewers: Vec::new(),
18263                fixer: None,
18264                conductor: conductor(),
18265                implementer_roster: Vec::new(),
18266                judge_roster: Vec::new(),
18267                reviewer_roster: Vec::new(),
18268            },
18269            sem: Arc::new(Semaphore::new(1)),
18270            pause: Pause::new(),
18271            interrupt: Pause::new(),
18272        }
18273    }
18274
18275    #[tokio::test]
18276    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18277        crate::run::pin_test_home();
18278        let temp = tempfile::tempdir().unwrap();
18279        let root = temp.path().canonicalize().unwrap();
18280        let repo = root.join("repo");
18281        std::fs::create_dir(&repo).unwrap();
18282        init_repo(&repo);
18283        git::git(&repo, &["config", "core.autocrlf", "false"])
18284            .await
18285            .unwrap();
18286        let mut runner = runner_at(RunStatus::Blocked);
18287        runner.state.repo = repo.clone();
18288        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18289        runner.state.config.agents = vec![conductor()];
18290        runner.state.config.graph.selected_implementer = Some("conductor".into());
18291        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18292        let (shell, command, setup_output) = if cfg!(windows) {
18293            (
18294                vec!["cmd".into(), "/C".into()],
18295                "echo ready>seat-ready",
18296                "ready\r\n",
18297            )
18298        } else {
18299            (
18300                vec!["sh".into(), "-c".into()],
18301                "printf ready > seat-ready",
18302                "ready",
18303            )
18304        };
18305        runner.state.config.verify.shell = Some(shell);
18306        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18307            run: Some(command.into()),
18308            ..Default::default()
18309        }];
18310        runner.state.save().unwrap();
18311        let path = runner.state.worktree_root().join("judge-1");
18312        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18313            .await
18314            .unwrap();
18315        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18316        git::git(&path, &["add", "."]).await.unwrap();
18317        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18318        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18319            .await
18320            .unwrap();
18321        git::git(&path, &["checkout", "--detach", &head])
18322            .await
18323            .unwrap();
18324        let evidence = crate::fold_evidence::snapshot(
18325            &runner.state,
18326            &crate::run::home().canonicalize().unwrap(),
18327            &head,
18328            64 * 1024 * 1024,
18329        )
18330        .await
18331        .unwrap();
18332        let mut contents = BTreeMap::new();
18333        contents.insert(
18334            repo.join("logs/HEAD"),
18335            format!("{head} {head} fixture\n").into_bytes(),
18336        );
18337        let record = serde_json::json!({
18338            "version":1, "run":runner.state.id, "source":path, "head":head,
18339            "evidence":evidence, "admin":{}, "admin_contents":contents,
18340            "history":{head.clone():evidence}, "phase":"evicted"
18341        });
18342        std::fs::write(
18343            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18344            serde_json::to_vec(&record).unwrap(),
18345        )
18346        .unwrap();
18347        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18348        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18349            .await
18350            .unwrap();
18351        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18352        assert!(
18353            !git::git_raw(&repo, &["cat-file", "-e", &head])
18354                .await
18355                .unwrap()
18356                .ok()
18357        );
18358        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18359        let error = resumed.execute().await.unwrap_err();
18360        assert!(
18361            error
18362                .to_string()
18363                .contains("selected implementation is held"),
18364            "{error:#}"
18365        );
18366        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18367        assert_eq!(
18368            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18369            setup_output
18370        );
18371        assert_eq!(
18372            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18373            "unrelated seat"
18374        );
18375        let error = resumed.execute().await.unwrap_err();
18376        assert!(
18377            error
18378                .to_string()
18379                .contains("selected implementation is held"),
18380            "{error:#}"
18381        );
18382        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18383        git::git(&path, &["add", "."]).await.unwrap();
18384        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18385        let next = git::git(
18386            &path,
18387            &[
18388                "commit-tree",
18389                &next_tree,
18390                "-p",
18391                &head,
18392                "-m",
18393                "next review checkpoint",
18394            ],
18395        )
18396        .await
18397        .unwrap();
18398        git::git(&path, &["checkout", "--detach", &next])
18399            .await
18400            .unwrap();
18401        resumed.state.reviews.push(
18402            serde_json::from_value(serde_json::json!({
18403                "round":1, "head":next, "reviews":[]
18404            }))
18405            .unwrap(),
18406        );
18407        resumed.state.save().unwrap();
18408        let error = resumed.execute().await.unwrap_err();
18409        assert!(
18410            error
18411                .to_string()
18412                .contains("selected implementation is held"),
18413            "{error:#}"
18414        );
18415        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18416    }
18417
18418    #[tokio::test]
18419    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18420        crate::run::pin_test_home();
18421        let mut runner = runner_at(RunStatus::Gating);
18422        runner.state.save().unwrap();
18423        let id = runner.state.id.clone();
18424        let passes = runner.state.passes.len();
18425        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18426        let mut claiming = Box::pin(runner.claim_driver());
18427        std::future::poll_fn(|cx| {
18428            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18429            std::task::Poll::Ready(())
18430        })
18431        .await;
18432        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18433        drop(owner);
18434        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18435            .await
18436            .unwrap()
18437            .unwrap();
18438        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18439        assert!(execution.runner.state.resource_block.is_none());
18440        assert_eq!(
18441            execution.runner.state.liveness(false),
18442            crate::run::Liveness::Live
18443        );
18444        drop(execution);
18445        assert!(RunState::load(&id).unwrap().driver_exited);
18446    }
18447
18448    #[tokio::test]
18449    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18450        crate::run::pin_test_home();
18451        let mut runner = runner_at(RunStatus::Gating);
18452        runner.state.save().unwrap();
18453        // run.json can commit, but the subsequent lease metadata write fails.
18454        // Cleanup must run after releasing the short ownership lock.
18455        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18456        assert!(runner.claim_driver().await.is_err());
18457        assert!(runner.state.driver_exited);
18458        assert_eq!(runner.state.driver_pid, None);
18459        let disk = RunState::load(&runner.state.id).unwrap();
18460        assert!(disk.driver_exited);
18461        assert_eq!(disk.driver_pid, None);
18462        assert_eq!(disk.status, RunStatus::Gating);
18463    }
18464
18465    #[tokio::test]
18466    async fn an_early_graph_error_releases_its_driver() {
18467        crate::run::pin_test_home();
18468        let mut runner = runner_at(RunStatus::Blocked);
18469        runner.state.config.graph.selected_implementer = Some("selected".into());
18470        runner.state.save().unwrap();
18471        assert!(runner.execute().await.is_err());
18472        let disk = RunState::load(&runner.state.id).unwrap();
18473        assert!(disk.driver_exited);
18474        assert_eq!(disk.driver_pid, None);
18475        assert_eq!(disk.status, RunStatus::Blocked);
18476    }
18477
18478    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18479        crate::run::MergeRepair {
18480            question: "repair-question".into(),
18481            head: "old-head".into(),
18482            pr: "https://github.com/o/r/pull/42".into(),
18483            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18484            gate: vec![outcome(Some(0), false)],
18485            gate_head: Some("old-head".into()),
18486            gate_fixes: Vec::new(),
18487            pre_gate: Vec::new(),
18488            phase,
18489            failure: None,
18490        }
18491    }
18492
18493    #[tokio::test]
18494    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18495        crate::run::pin_test_home();
18496        let mut runner = runner_at(RunStatus::Landing);
18497        runner.state.merge_repairs.push(merge_repair_request(
18498            crate::run::MergeRepairPhase::Requested,
18499        ));
18500        runner.state.save().unwrap();
18501        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18502        assert!(
18503            runner
18504                .execute()
18505                .await
18506                .unwrap_err()
18507                .to_string()
18508                .contains("already running")
18509        );
18510        assert_eq!(
18511            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18512            crate::run::MergeRepairPhase::Requested
18513        );
18514        drop(claim);
18515        let mut disk = runner.state.clone();
18516        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18517        disk.save().unwrap();
18518        assert!(
18519            runner
18520                .execute()
18521                .await
18522                .unwrap_err()
18523                .to_string()
18524                .contains("decision changed")
18525        );
18526        assert_eq!(
18527            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18528            crate::run::MergeRepairPhase::Complete
18529        );
18530    }
18531
18532    #[test]
18533    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18534        crate::run::pin_test_home();
18535        let mut runner = runner_at(RunStatus::Landing);
18536        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18537        runner.state.reviews = request.reviews.clone();
18538        runner.state.gate = request.gate.clone();
18539        runner.state.gate_ran = true;
18540        runner.state.land_approval = Some(crate::run::LandApproval {
18541            question: "old".into(),
18542            head: "old-head".into(),
18543        });
18544        runner.state.merge_repairs.push(request);
18545        runner.begin_merge_repair_review(0).unwrap();
18546        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18547        assert!(!runner.state.gate_status().ok());
18548        assert!(runner.state.land_approval.is_none());
18549        let disk = RunState::load(&runner.state.id).unwrap();
18550        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18551        assert!(disk.merge_repairs[0].gate[0].ok());
18552        assert_eq!(
18553            disk.merge_repairs[0].phase,
18554            crate::run::MergeRepairPhase::Reviewing
18555        );
18556        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18557        assert!(disk.reviews.is_empty());
18558    }
18559
18560    #[tokio::test]
18561    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18562        crate::run::pin_test_home();
18563        for phase in [
18564            crate::run::MergeRepairPhase::Fixing,
18565            crate::run::MergeRepairPhase::Requested,
18566        ] {
18567            let mut runner = runner_at(RunStatus::Landing);
18568            runner.state.config.graph.land_rounds = 0;
18569            runner.state.merge_repairs.push(merge_repair_request(phase));
18570            runner.state.save().unwrap();
18571            assert!(!runner.repair_merge_review().await.unwrap());
18572            assert_eq!(runner.state.status, RunStatus::Blocked);
18573            assert_eq!(
18574                runner.state.merge_repairs[0].phase,
18575                crate::run::MergeRepairPhase::Failed
18576            );
18577            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18578            assert!(
18579                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18580                    "interrupted"
18581                } else {
18582                    "budget exhausted"
18583                })
18584            );
18585            assert!(runner.state.merge.is_none());
18586        }
18587    }
18588
18589    /// `park_here` folding in the reason `Pause::park_because` recorded -
18590    /// this is what lets an operator reading a run's events tell an
18591    /// interrupt-driven park from an ordinary shutdown park.
18592    #[test]
18593    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18594        crate::run::pin_test_home();
18595        let mut runner = runner_at(RunStatus::Implementing);
18596        let interrupt = Pause::new();
18597        runner.watch_interrupt(interrupt.clone());
18598
18599        interrupt.park_because("task a1b2 asked to run first");
18600
18601        assert!(runner.park_here().expect("park_here"));
18602        assert!(runner.state.parked);
18603        let last = runner.state.events.last().expect("a park event");
18604        assert_eq!(last.node, "park");
18605        assert!(
18606            last.message.contains("task a1b2 asked to run first"),
18607            "expected the interrupt reason in {:?}",
18608            last.message
18609        );
18610    }
18611
18612    /// The cause is stored on the pass when it parks, and the first cause a
18613    /// shared pause was given is the one that stays.
18614    #[test]
18615    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18616        crate::run::pin_test_home();
18617        let mut runner = runner_at(RunStatus::Implementing);
18618        runner.state.begin_pass();
18619        let shutdown = Pause::new();
18620        runner.on_pause(shutdown.clone());
18621        let upgrade = ParkCause::Upgrade {
18622            from: "0.121.0".to_owned(),
18623            to: Some("0.121.1".to_owned()),
18624        };
18625        shutdown.park_for(upgrade.clone());
18626        shutdown.park_for(ParkCause::Operator);
18627
18628        assert!(runner.park_here().expect("park_here"));
18629        assert_eq!(
18630            runner.state.passes[0].end,
18631            Some(crate::run::PassEnd::Parked { cause: upgrade })
18632        );
18633        let last = runner.state.events.last().expect("a park event");
18634        assert!(last.message.contains("0.121.0"), "{}", last.message);
18635    }
18636
18637    #[tokio::test]
18638    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18639        crate::run::pin_test_home();
18640        let mut runner = runner_at(RunStatus::Reviewing);
18641        runner.state.save().unwrap();
18642        let mut execution = runner.claim_driver().await.expect("first execution");
18643        assert_eq!(execution.runner.state.passes.len(), 1);
18644        execution
18645            .driver
18646            .finish(&mut execution.runner.state)
18647            .await
18648            .unwrap();
18649    }
18650
18651    #[tokio::test]
18652    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18653        crate::run::pin_test_home();
18654        let mut runner = runner_at(RunStatus::Reviewing);
18655        runner.state.config.agents = vec![conductor()];
18656        runner.state.config.graph.timeout_resource_wait = 0;
18657        runner.state.save().unwrap();
18658        let id = runner.state.id.clone();
18659        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18660        assert!(runner.claim_driver().await.is_err());
18661        drop(owner);
18662        drop(runner);
18663        let mut resumed = Runner::resume(&id).unwrap();
18664        assert_eq!(resumed.state.driver_claimed, Some(false));
18665        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18666        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18667        let mut execution = resumed
18668            .claim_driver()
18669            .await
18670            .expect("retry initial claim after reload");
18671        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18672        assert_eq!(execution.runner.state.passes.len(), 1);
18673        execution
18674            .driver
18675            .finish(&mut execution.runner.state)
18676            .await
18677            .unwrap();
18678    }
18679
18680    #[tokio::test]
18681    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18682        crate::run::pin_test_home();
18683        let mut runner = runner_at(RunStatus::Reviewing);
18684        runner.state.config.agents = vec![conductor()];
18685        runner.state.save().unwrap();
18686        let id = runner.state.id.clone();
18687        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18688        let mut resumed = Runner::resume(&id).unwrap();
18689        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18690        assert!(resumed.claim_driver().await.is_err());
18691        assert!(RunState::load(&id).unwrap().passes.is_empty());
18692        drop(lease);
18693        let mut execution = resumed.claim_driver().await.unwrap();
18694        execution
18695            .driver
18696            .finish(&mut execution.runner.state)
18697            .await
18698            .unwrap();
18699    }
18700
18701    #[tokio::test]
18702    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18703        crate::run::pin_test_home();
18704        let mut runner = runner_at(RunStatus::Reviewing);
18705        let state = &mut runner.state;
18706        state.config.agents = vec![conductor()];
18707        state.driver_claimed = None;
18708        state.driver_pid = Some(std::process::id());
18709        state.driver_started_at = None;
18710        state.begin_pass();
18711        state.save().unwrap();
18712        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18713        std::fs::write(
18714            state.dir().join("driver.json"),
18715            serde_json::to_vec(&serde_json::json!({
18716                "pid": state.driver_pid, "started_at": null,
18717                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18718            }))
18719            .unwrap(),
18720        )
18721        .unwrap();
18722        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18723        drop(lease);
18724        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18725        let mut resumed = Runner::resume(&state.id).unwrap();
18726        assert!(!resumed.state.driver_exited);
18727        let mut execution = resumed
18728            .claim_driver()
18729            .await
18730            .expect("lease proves previous execution ended");
18731        assert_eq!(execution.runner.state.passes.len(), 2);
18732        execution
18733            .driver
18734            .finish(&mut execution.runner.state)
18735            .await
18736            .unwrap();
18737    }
18738
18739    #[tokio::test]
18740    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18741        crate::run::pin_test_home();
18742        let mut runner = runner_at(RunStatus::Reviewing);
18743        let mut json = serde_json::to_value(&runner.state).unwrap();
18744        json.as_object_mut().unwrap().remove("driver_claimed");
18745        runner.state = serde_json::from_value(json).unwrap();
18746        assert_eq!(runner.state.driver_claimed, None);
18747        runner.state.save().unwrap();
18748        let mut execution = runner
18749            .claim_driver()
18750            .await
18751            .expect("the execution lease, not a missing marker, excludes a second driver");
18752        assert_eq!(execution.runner.state.passes.len(), 1);
18753        execution
18754            .driver
18755            .finish(&mut execution.runner.state)
18756            .await
18757            .unwrap();
18758    }
18759
18760    #[tokio::test]
18761    async fn driver_admission_refuses_a_proven_live_driver() {
18762        crate::run::pin_test_home();
18763        let mut runner = runner_at(RunStatus::Reviewing);
18764        let pid = std::process::id();
18765        runner.state.driver_pid = Some(pid);
18766        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18767        assert!(runner.state.driver_started_at.is_some());
18768        runner.state.save().unwrap();
18769        let error = runner
18770            .claim_driver()
18771            .await
18772            .err()
18773            .expect("a live driver must refuse");
18774        assert!(error.to_string().contains("live driver"), "{error:#}");
18775        let saved = RunState::load(&runner.state.id).unwrap();
18776        assert_eq!(saved.driver_pid, Some(pid));
18777        assert_eq!(saved.status, RunStatus::Reviewing);
18778        assert!(saved.passes.is_empty());
18779    }
18780
18781    #[test]
18782    fn parking_resource_halt_persists_the_pass_once() {
18783        crate::run::pin_test_home();
18784        let mut runner = runner_at(RunStatus::Reviewing);
18785        runner.state.begin_pass();
18786        runner.state.resource_block = Some(crate::run::ResourceBlock {
18787            node: "fix".into(),
18788            kind: "shared build cache".into(),
18789            detail: "cancelled".into(),
18790            at: Timestamp::now(),
18791            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18792        });
18793        let cause = ParkCause::Upgrade {
18794            from: "old".into(),
18795            to: Some("new".into()),
18796        };
18797        runner.pause.park_for(cause.clone());
18798        // Both direct fixer exits and graph resource boundaries must complete
18799        // parking, rather than returning a stale Reviewing record.
18800        runner.stop_for_resource().unwrap();
18801        assert!(runner.resource_halt().unwrap());
18802        assert!(runner.park_here().unwrap());
18803        let saved = RunState::load(&runner.state.id).unwrap();
18804        assert!(saved.parked);
18805        assert_eq!(saved.status, RunStatus::Reviewing);
18806        assert!(saved.passes[0].ended_at.is_some());
18807        assert_eq!(
18808            saved.passes[0].end,
18809            Some(crate::run::PassEnd::Parked {
18810                cause: cause.clone()
18811            })
18812        );
18813        assert_eq!(
18814            saved.resource_block.unwrap().cause,
18815            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18816        );
18817        assert_eq!(
18818            saved
18819                .events
18820                .iter()
18821                .filter(|event| event.node == "park")
18822                .count(),
18823            1
18824        );
18825    }
18826
18827    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
18828    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
18829    /// cleared) must not make a *different* run - one only watching its own,
18830    /// unshared interrupt `Pause` - see itself as parked. If a future change
18831    /// ever collapsed these back into one handle, the interrupt scheduler
18832    /// would park every run for the rest of the daemon's life, not just the
18833    /// one it meant to interrupt.
18834    #[test]
18835    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18836        crate::run::pin_test_home();
18837        let mut runner = runner_at(RunStatus::Implementing);
18838        let shutdown = Pause::new();
18839        runner.on_pause(shutdown.clone());
18840        let interrupt = Pause::new();
18841        runner.watch_interrupt(interrupt.clone());
18842
18843        // Nobody has asked for anything yet.
18844        assert!(!runner.park_here().expect("park_here"));
18845        assert!(!runner.state.parked);
18846
18847        // Only the interrupt handle fires; the shutdown handle stays clear.
18848        interrupt.park_because("test");
18849        assert!(!shutdown.parked());
18850        assert!(runner.park_here().expect("park_here"));
18851    }
18852
18853    /// The property every prior attempt at this feature failed to pin down:
18854    /// asking a run to park while one of its nodes has a real, in-flight
18855    /// async operation running (an agent call, in production) must not cut
18856    /// that operation short. `park_here` is only ever consulted *between*
18857    /// `execute`'s node calls - see its own doc - so nothing inside a node
18858    /// can observe a park request until the node itself returns. This proves
18859    /// that structurally, with real `tokio` concurrency and a channel
18860    /// handshake (never a sleep, which would only prove "usually", not
18861    /// "cannot"): the "node" below reports that it has genuinely started,
18862    /// and only then is the park requested; the node still has to be told to
18863    /// finish before `park_here` is ever called, exactly mirroring every
18864    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
18865    /// in `execute`.
18866    #[tokio::test]
18867    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18868        crate::run::pin_test_home();
18869        let mut runner = runner_at(RunStatus::Implementing);
18870        let interrupt = Pause::new();
18871        runner.watch_interrupt(interrupt.clone());
18872
18873        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18874        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18875
18876        // Stands in for one node's in-flight agent call: it proves it has
18877        // genuinely started, then blocks - exactly as a spawned CLI process
18878        // does - until told to finish.
18879        let node = async move {
18880            started_tx.send(()).expect("send started");
18881            finish_rx.await.expect("recv finish");
18882            "node finished"
18883        };
18884
18885        let interrupter = async move {
18886            started_rx.await.expect("recv started");
18887            // The call is now genuinely in flight. Ask it to park.
18888            interrupt.park_because("higher-priority task waiting");
18889            // Nothing the node does can observe this yet - there is no
18890            // check inside it, by construction - so let the executor run
18891            // anything pending and then let the node finish on its own.
18892            tokio::task::yield_now().await;
18893            finish_tx.send(()).expect("send finish");
18894        };
18895
18896        let (node_result, ()) = tokio::join!(node, interrupter);
18897        assert_eq!(
18898            node_result, "node finished",
18899            "the in-flight call ran to completion"
18900        );
18901
18902        // Only now, at the boundary the real `execute` would check right
18903        // after this node, does the park take effect.
18904        assert!(runner.park_here().expect("park_here"));
18905        assert!(runner.state.parked);
18906    }
18907
18908    /// A run parked mid-competition carries every field it had accumulated
18909    /// through the exact same disk round-trip an ordinary resume uses -
18910    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
18911    /// Nothing about parking for an interrupt is a special case of that path;
18912    /// this is what proves it rather than assuming it.
18913    #[test]
18914    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18915        crate::run::pin_test_home();
18916        let mut runner = runner_at(RunStatus::Judging);
18917        // `Runner::resume` re-resolves roles from the saved config, which
18918        // refuses an empty roster - give it the same minimal one `conductor`
18919        // itself uses.
18920        runner.state.config.agents = vec![conductor()];
18921        runner.state.candidates = vec![Candidate {
18922            index: 0,
18923            label: 'A',
18924            agent: "alpha".to_owned(),
18925            branch: "magi/x/A".to_owned(),
18926            worktree: PathBuf::from("/nonexistent/worktree"),
18927            summary: "did the thing".to_owned(),
18928            stat: "1 file changed".to_owned(),
18929            files: 1,
18930            commits: 1,
18931            empty: false,
18932            failed: None,
18933            verified_noop: None,
18934            duration_ms: 1234,
18935            folded: false,
18936        }];
18937        let run_id = runner.state.id.clone();
18938
18939        let interrupt = Pause::new();
18940        runner.watch_interrupt(interrupt.clone());
18941        interrupt.park_because("task c3d4 asked to run first");
18942        assert!(runner.park_here().expect("park_here"));
18943
18944        let resumed = Runner::resume(&run_id).expect("resume");
18945        assert_eq!(resumed.state.candidates.len(), 1);
18946        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18947        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18948        assert_eq!(resumed.state.status, runner.state.status);
18949        assert!(
18950            resumed.state.parked,
18951            "still parked until `execute` actually walks the graph again"
18952        );
18953        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18954    }
18955
18956    /// A fresh open question on `run`, stored and handed back for assertions.
18957    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18958        let mut q = ask::Question::new(
18959            run.to_owned(),
18960            "implement".to_owned(),
18961            "impl-A".to_owned(),
18962            "Which storage backend should the cache use?".to_owned(),
18963            String::new(),
18964            vec!["SQLite".to_owned(), "Redis".to_owned()],
18965        );
18966        store.put(&mut q).unwrap();
18967        q
18968    }
18969
18970    #[test]
18971    fn a_failed_runs_open_question_is_abandoned() {
18972        ask_test_home();
18973        let store = ask::Questions::open();
18974        let mut runner = runner_at(RunStatus::Failed);
18975        let run = runner.state.id.clone();
18976        let q = ask_open_question(&store, &run);
18977
18978        runner.settle_questions();
18979
18980        let back = store.get(&q.id).unwrap();
18981        assert!(
18982            !back.status.open(),
18983            "the seat that asked died with the run; nobody is left to read an answer"
18984        );
18985        assert!(
18986            back.detail.contains(&run) && back.detail.contains("failed"),
18987            "the reason names what the run became, not just that it is gone: {}",
18988            back.detail
18989        );
18990    }
18991
18992    #[test]
18993    fn a_merged_runs_open_question_is_abandoned_too() {
18994        ask_test_home();
18995        let store = ask::Questions::open();
18996        // A run that finishes cleanly still leaves nobody to read an answer -
18997        // this is not only a failure-path cleanup.
18998        for status in [RunStatus::Merged, RunStatus::Ready] {
18999            let mut runner = runner_at(status);
19000            let run = runner.state.id.clone();
19001            let q = ask_open_question(&store, &run);
19002
19003            runner.settle_questions();
19004
19005            let back = store.get(&q.id).unwrap();
19006            assert!(
19007                !back.status.open(),
19008                "{status:?} run's question must not outlive the run"
19009            );
19010        }
19011    }
19012
19013    #[test]
19014    fn a_still_resumable_runs_open_question_is_left_alone() {
19015        ask_test_home();
19016        let store = ask::Questions::open();
19017        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19018        // review round and the seat sessions are all still on disk — so a
19019        // question asked mid-round may yet get a real answer from a real
19020        // resume. Sweeping it here would be exactly the failure mode this
19021        // whole feature exists to avoid on the other side.
19022        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19023            let mut runner = runner_at(status);
19024            let run = runner.state.id.clone();
19025            let q = ask_open_question(&store, &run);
19026
19027            runner.settle_questions();
19028
19029            let back = store.get(&q.id).unwrap();
19030            assert!(
19031                back.status.open(),
19032                "{status:?} is still alive; the question must still be waiting"
19033            );
19034        }
19035    }
19036
19037    #[test]
19038    fn settle_questions_never_touches_an_already_answered_question() {
19039        ask_test_home();
19040        let store = ask::Questions::open();
19041        let mut runner = runner_at(RunStatus::Failed);
19042        let run = runner.state.id.clone();
19043        let mut q = ask_open_question(&store, &run);
19044        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19045            .unwrap();
19046        store.put(&mut q).unwrap();
19047
19048        // Called twice, the way a crash-recovered daemon reclaim and the
19049        // graph's own cleanup both can for the same run — `abandon_for_run`
19050        // only ever touches what is still open, so this must be inert both
19051        // times, not merely the second.
19052        runner.settle_questions();
19053        runner.settle_questions();
19054
19055        let back = store.get(&q.id).unwrap();
19056        assert_eq!(
19057            back.status,
19058            ask::QuestionStatus::Answered,
19059            "a real answer is a decision on record, never overwritten by a sweep"
19060        );
19061    }
19062
19063    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19064    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19065    /// without merging, whose winner is still the operator's answer to read.
19066    /// Nothing previously called `fold_run` itself with a real `tally`, so
19067    /// this is the first test to pin down the one distinction the whole
19068    /// automatic-fold feature depends on: the winner's worktree and branch
19069    /// must survive, everything else sharing the run's worktree bay - a
19070    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19071    /// second sweep treats every non-winner directory under the bay alike -
19072    /// must not.
19073    #[tokio::test]
19074    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19075        crate::run::pin_test_home();
19076        let tmp = tempfile::tempdir().expect("tempdir");
19077        let repo = tmp.path().join("repo");
19078        std::fs::create_dir_all(&repo).unwrap();
19079        init_repo(&repo);
19080
19081        let mut config = Config::default();
19082        config.graph.worktree_root = Some(tmp.path().join("wt"));
19083
19084        let mut state = RunState::new(
19085            repo.clone(),
19086            "main".to_owned(),
19087            "deadbeef".to_owned(),
19088            "task".to_owned(),
19089            config,
19090        );
19091        let root = state.worktree_root();
19092        let wt_a = root.join("cand-A");
19093        let wt_b = root.join("cand-B");
19094        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19095            .await
19096            .expect("worktree A");
19097        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19098            .await
19099            .expect("worktree B");
19100
19101        state.candidates = vec![
19102            Candidate {
19103                index: 0,
19104                label: 'A',
19105                agent: "alpha".to_owned(),
19106                branch: "magi/x/A".to_owned(),
19107                worktree: wt_a.clone(),
19108                summary: String::new(),
19109                stat: String::new(),
19110                files: 0,
19111                commits: 0,
19112                empty: false,
19113                failed: None,
19114                verified_noop: None,
19115                duration_ms: 0,
19116                folded: false,
19117            },
19118            Candidate {
19119                index: 1,
19120                label: 'B',
19121                agent: "beta".to_owned(),
19122                branch: "magi/x/B".to_owned(),
19123                worktree: wt_b.clone(),
19124                summary: String::new(),
19125                stat: String::new(),
19126                files: 0,
19127                commits: 0,
19128                empty: false,
19129                failed: None,
19130                verified_noop: None,
19131                duration_ms: 0,
19132                folded: false,
19133            },
19134        ];
19135        state.tally = Some(Tally {
19136            first_choice: BTreeMap::from([('A', 1)]),
19137            borda: BTreeMap::new(),
19138            winner: 'A',
19139            rankings: 1,
19140            unanimous_initial: true,
19141            deliberated: false,
19142            changed_votes: 0,
19143            unanimous_final: true,
19144            tie_break: None,
19145            judges: 1,
19146            present: 1,
19147            quorum: 1,
19148            met_quorum: true,
19149            uncontested: None,
19150        });
19151        state.status = RunStatus::Ready;
19152
19153        fold_run(&mut state, false, &crate::run::home())
19154            .await
19155            .expect("fold_run");
19156
19157        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19158        assert!(
19159            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19160            "the unmerged winner's branch survives"
19161        );
19162        assert!(
19163            !state.candidates[0].folded,
19164            "the winner is not marked folded"
19165        );
19166
19167        assert!(!wt_b.exists(), "the loser's worktree is removed");
19168        assert!(
19169            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19170            "the loser's branch is removed"
19171        );
19172        assert!(state.candidates[1].folded, "the loser is marked folded");
19173    }
19174
19175    #[tokio::test]
19176    #[cfg(unix)]
19177    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19178        use std::os::fd::AsRawFd as _;
19179        use std::time::Duration;
19180
19181        let tmp = tempfile::tempdir().unwrap();
19182        let repo = tmp.path().join("repo");
19183        std::fs::create_dir_all(&repo).unwrap();
19184        init_repo(&repo);
19185        let common = repo.join(".git");
19186        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19187        let home = tmp.path().join("home");
19188        let mut config = Config::default();
19189        config.coordination = crate::config::test_coordination();
19190        config.graph.worktree_root = Some(tmp.path().join("wt"));
19191        config.graph.timeout_resource_wait = 1;
19192        let mut state = RunState::new(
19193            repo.clone(),
19194            "main".into(),
19195            "deadbeef".into(),
19196            "task".into(),
19197            config,
19198        );
19199        let root = state.worktree_root();
19200        let candidate = root.join("cand-A");
19201        let reviewer = root.join("review-1");
19202        let branch = state.branch_for('A');
19203        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19204            .await
19205            .unwrap();
19206        git::worktree_add_detached(&repo, &reviewer, "main")
19207            .await
19208            .unwrap();
19209        git::acquire_worktree_config(&repo).await.unwrap();
19210        state.enabled_worktree_config = true;
19211        state.status = RunStatus::Ready;
19212        state.candidates.push(Candidate {
19213            index: 0,
19214            label: 'A',
19215            agent: "implement".into(),
19216            branch: branch.clone(),
19217            worktree: candidate.clone(),
19218            summary: String::new(),
19219            stat: String::new(),
19220            files: 0,
19221            commits: 0,
19222            empty: false,
19223            failed: None,
19224            verified_noop: None,
19225            duration_ms: 0,
19226            folded: false,
19227        });
19228        state.save_under(&home).unwrap();
19229        let config_before = std::fs::read(common.join("config")).unwrap();
19230        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19231            .await
19232            .unwrap();
19233        // A separate OS handle bypasses the launcher's local gate, exactly as
19234        // an external updater does, and must still prevent all folding work.
19235        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19236        assert_eq!(
19237            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19238            0
19239        );
19240        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19241        assert!(
19242            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19243                .await
19244                .is_err()
19245        );
19246        assert!(candidate.exists());
19247        assert!(reviewer.exists());
19248        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19249        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19250        assert_eq!(
19251            git::git(&repo, &["worktree", "list", "--porcelain"])
19252                .await
19253                .unwrap(),
19254            worktrees_before
19255        );
19256        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19257            .await
19258            .unwrap()
19259            .unwrap_err();
19260        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19261        drop(fold);
19262        assert!(candidate.exists() && reviewer.exists());
19263        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19264        assert_eq!(
19265            state.resource_block.as_ref().unwrap().kind,
19266            "repository coordination"
19267        );
19268        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19269        drop(external);
19270        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19271        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19272            .await
19273            .unwrap()
19274            .unwrap();
19275        drop(fold);
19276        assert!(removed.contains(&branch));
19277        assert!(!root.exists());
19278        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19279        assert!(state.candidates[0].folded);
19280        assert!(!state.enabled_worktree_config);
19281        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19282        let enabled = git::git_raw(
19283            &repo,
19284            &["config", "--local", "--get", "extensions.worktreeConfig"],
19285        )
19286        .await
19287        .unwrap();
19288        assert!(!enabled.ok(), "the last config reference was released");
19289    }
19290
19291    /// A branch handed to a later run is that run's (and its pull request's):
19292    /// folding the run that released it must not delete it.
19293    #[tokio::test]
19294    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19295        let tmp = tempfile::tempdir().expect("tempdir");
19296        let repo = tmp.path().join("repo");
19297        std::fs::create_dir_all(&repo).unwrap();
19298        init_repo(&repo);
19299        let home = tmp.path().join("home");
19300
19301        let mut config = Config::default();
19302        config.graph.worktree_root = Some(tmp.path().join("wt"));
19303        let mut state = RunState::new(
19304            repo.clone(),
19305            "main".to_owned(),
19306            "deadbeef".to_owned(),
19307            "task".to_owned(),
19308            config,
19309        );
19310        // The worktree is already gone (released); the branch survives.
19311        git::git(&repo, &["branch", "magi/x/A", "main"])
19312            .await
19313            .expect("branch");
19314        state.candidates = vec![Candidate {
19315            index: 0,
19316            label: 'A',
19317            agent: "alpha".to_owned(),
19318            branch: "magi/x/A".to_owned(),
19319            worktree: state.worktree_root().join("cand-A"),
19320            summary: String::new(),
19321            stat: String::new(),
19322            files: 0,
19323            commits: 0,
19324            empty: false,
19325            failed: None,
19326            verified_noop: None,
19327            duration_ms: 0,
19328            folded: true,
19329        }];
19330        state.released_to = Some("20260901-000000-new1".to_owned());
19331        state.released_branches = vec!["magi/x/A".to_owned()];
19332
19333        fold_run(&mut state, true, &home).await.expect("fold_run");
19334        state.status = RunStatus::Merged;
19335        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19336        state.merge = Some(crate::run::MergeOutcome {
19337            mode: MergeMode::Pr,
19338            ok: true,
19339            detail: "confirmed merged".into(),
19340            empty: false,
19341        });
19342        fold_run(&mut state, true, &home)
19343            .await
19344            .expect("merged fold");
19345
19346        assert!(
19347            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19348            "the handed-over branch survives a fold"
19349        );
19350    }
19351
19352    /// A winner with nothing ahead of the base is caught before `gh` is ever
19353    /// asked for a pull request, and the message carries what the task's
19354    /// references resolved to.
19355    #[tokio::test]
19356    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19357        let tmp = tempfile::tempdir().expect("tempdir");
19358        let repo = tmp.path().join("repo");
19359        std::fs::create_dir_all(&repo).unwrap();
19360        init_repo(&repo);
19361        let run = |args: &[&str]| {
19362            let out = std::process::Command::new("git")
19363                .quiet()
19364                .args(args)
19365                .current_dir(&repo)
19366                .output()
19367                .expect("spawn git");
19368            assert!(out.status.success(), "git {args:?}");
19369        };
19370        run(&["branch", "magi/x/A"]);
19371        run(&["checkout", "-q", "-b", "magi/x/B"]);
19372        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19373        run(&["add", "-A"]);
19374        run(&["commit", "-q", "-m", "work"]);
19375        run(&["checkout", "-q", "main"]);
19376
19377        // A pull request is compared against the remote's base, so the
19378        // fixture needs one. Before it exists nothing can be read, and the
19379        // answer must be "not empty".
19380        let probe = RunState::new(
19381            repo.clone(),
19382            "main".to_owned(),
19383            "deadbeef".to_owned(),
19384            "task".to_owned(),
19385            Config::default(),
19386        );
19387        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19388        let bare = tmp.path().join("origin.git");
19389        let out = std::process::Command::new("git")
19390            .quiet()
19391            .args(["init", "-q", "--bare"])
19392            .arg(&bare)
19393            .output()
19394            .expect("spawn git");
19395        assert!(out.status.success(), "git init --bare");
19396        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19397        run(&["push", "-q", "origin", "main"]);
19398
19399        let mut state = RunState::new(
19400            repo.clone(),
19401            "main".to_owned(),
19402            "deadbeef".to_owned(),
19403            "task".to_owned(),
19404            Config::default(),
19405        );
19406        state.seeds = vec![refs::Seed {
19407            token: "magi/27b2/A".to_owned(),
19408            kind: refs::SeedKind::Unresolved,
19409            sha: String::new(),
19410            branch: true,
19411            detail: "no branch or commit named magi/27b2/A".to_owned(),
19412        }];
19413
19414        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19415        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19416        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19417        let detail = empty_candidate_detail(&state, "main");
19418        assert!(detail.starts_with("empty candidate"), "{detail}");
19419        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19420        assert!(detail.contains("--start-from"), "{detail}");
19421    }
19422
19423    /// `status == Ready` used to be read as "this is the harmless
19424    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19425    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19426    /// run's PR was closed without merging — and reentering `merge` with
19427    /// `mode` still `Pr` does not know the difference, so it pushed and
19428    /// opened a second pull request. `mode == Local` reproduces the same
19429    /// blind spot without a network call: reentry must not attempt another
19430    /// git merge once this node has already recorded an outcome.
19431    #[tokio::test]
19432    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19433        let tmp = tempfile::tempdir().expect("tempdir");
19434        let repo = tmp.path().join("repo");
19435        std::fs::create_dir_all(&repo).unwrap();
19436        init_repo(&repo);
19437
19438        let mut config = Config::default();
19439        config.merge.mode = MergeMode::Local;
19440
19441        let mut state = RunState::new(
19442            repo.clone(),
19443            "main".to_owned(),
19444            "deadbeef".to_owned(),
19445            "task".to_owned(),
19446            config,
19447        );
19448        state.candidates = vec![Candidate {
19449            index: 0,
19450            label: 'A',
19451            agent: "alpha".to_owned(),
19452            branch: "does-not-exist".to_owned(),
19453            worktree: repo.clone(),
19454            summary: String::new(),
19455            stat: String::new(),
19456            files: 0,
19457            commits: 0,
19458            empty: false,
19459            failed: None,
19460            verified_noop: None,
19461            duration_ms: 0,
19462            folded: false,
19463        }];
19464        state.tally = Some(Tally {
19465            first_choice: BTreeMap::from([('A', 1)]),
19466            borda: BTreeMap::new(),
19467            winner: 'A',
19468            rankings: 1,
19469            unanimous_initial: true,
19470            deliberated: false,
19471            changed_votes: 0,
19472            unanimous_final: true,
19473            tie_break: None,
19474            judges: 0,
19475            present: 0,
19476            quorum: 0,
19477            met_quorum: true,
19478            uncontested: Some("only candidate A produced a change".to_owned()),
19479        });
19480        state.reviews = vec![ReviewRound {
19481            round: 1,
19482            head: "deadbeef".to_owned(),
19483            verified_head: None,
19484            verified_at: None,
19485            reviews: Vec::new(),
19486            e2e: Vec::new(),
19487            fix: None,
19488            blocking: 0,
19489            answered: 0,
19490            expected: 0,
19491            clean: true,
19492            verify_retried: false,
19493            e2e_deferred: false,
19494            e2e_defer_reason: None,
19495            progressed: false,
19496            vote_split: false,
19497            reconsideration: Vec::new(),
19498            verdict: None,
19499        }];
19500        state.gate = vec![CommandOutcome {
19501            command: "test".to_owned(),
19502            code: Some(0),
19503            output_tail: String::new(),
19504            duration_ms: 0,
19505            resource_blocked: false,
19506        }];
19507        state.gate_ran = true;
19508        // Reached its conclusion already — e.g. `land` closing the PR without
19509        // merging it, which (like the honest `MergeMode::None` path) leaves
19510        // `status` at `Ready`. The recorded outcome is what actually marks
19511        // this node done.
19512        state.status = RunStatus::Ready;
19513        state.merge = Some(MergeOutcome {
19514            mode: MergeMode::Local,
19515            ok: false,
19516            detail: "already concluded".to_owned(),
19517            empty: false,
19518        });
19519
19520        let mut runner = Runner {
19521            state,
19522            roles: ResolvedRoles {
19523                implementers: Vec::new(),
19524                judges: Vec::new(),
19525                reviewers: Vec::new(),
19526                fixer: None,
19527                conductor: conductor(),
19528                implementer_roster: Vec::new(),
19529                judge_roster: Vec::new(),
19530                reviewer_roster: Vec::new(),
19531            },
19532            sem: Arc::new(Semaphore::new(1)),
19533            pause: Pause::new(),
19534            interrupt: Pause::new(),
19535        };
19536
19537        runner.merge().await.expect("merge");
19538
19539        assert_eq!(
19540            runner.state.status,
19541            RunStatus::Ready,
19542            "a concluded run's status must not change on reentry"
19543        );
19544        assert_eq!(
19545            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19546            Some("already concluded"),
19547            "merge must not run again once the node already recorded an outcome"
19548        );
19549    }
19550
19551    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19552    /// when its last attempt was resource-blocked (the shared build cache
19553    /// could not be acquired or confirmed fresh in time - see
19554    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19555    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19556    /// a run merge a tree the gate never actually checked - exactly the case
19557    /// a contended cache produces on every retry until it clears. `merge`
19558    /// must refuse until `gate` has actually recorded an attempt.
19559    #[tokio::test]
19560    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19561        let tmp = tempfile::tempdir().expect("tempdir");
19562        let repo = tmp.path().join("repo");
19563        std::fs::create_dir_all(&repo).unwrap();
19564        init_repo(&repo);
19565
19566        let mut config = Config::default();
19567        config.merge.mode = MergeMode::Local;
19568
19569        let mut state = RunState::new(
19570            repo.clone(),
19571            "main".to_owned(),
19572            "deadbeef".to_owned(),
19573            "task".to_owned(),
19574            config,
19575        );
19576        state.candidates = vec![Candidate {
19577            index: 0,
19578            label: 'A',
19579            agent: "alpha".to_owned(),
19580            branch: "does-not-exist".to_owned(),
19581            worktree: repo.clone(),
19582            summary: String::new(),
19583            stat: String::new(),
19584            files: 0,
19585            commits: 0,
19586            empty: false,
19587            failed: None,
19588            verified_noop: None,
19589            duration_ms: 0,
19590            folded: false,
19591        }];
19592        state.tally = Some(Tally {
19593            first_choice: BTreeMap::from([('A', 1)]),
19594            borda: BTreeMap::new(),
19595            winner: 'A',
19596            rankings: 1,
19597            unanimous_initial: true,
19598            deliberated: false,
19599            changed_votes: 0,
19600            unanimous_final: true,
19601            tie_break: None,
19602            judges: 0,
19603            present: 0,
19604            quorum: 0,
19605            met_quorum: true,
19606            uncontested: Some("only candidate A produced a change".to_owned()),
19607        });
19608        state.reviews = vec![ReviewRound {
19609            round: 1,
19610            head: "deadbeef".to_owned(),
19611            verified_head: None,
19612            verified_at: None,
19613            reviews: Vec::new(),
19614            e2e: Vec::new(),
19615            fix: None,
19616            blocking: 0,
19617            answered: 0,
19618            expected: 0,
19619            clean: true,
19620            verify_retried: false,
19621            e2e_deferred: false,
19622            e2e_defer_reason: None,
19623            progressed: false,
19624            vote_split: false,
19625            reconsideration: Vec::new(),
19626            verdict: None,
19627        }];
19628        // The point: `gate` has not recorded anything yet.
19629        state.gate = Vec::new();
19630        state.gate_ran = false;
19631        state.status = RunStatus::Gating;
19632
19633        let mut runner = Runner {
19634            state,
19635            roles: ResolvedRoles {
19636                implementers: Vec::new(),
19637                judges: Vec::new(),
19638                reviewers: Vec::new(),
19639                fixer: None,
19640                conductor: conductor(),
19641                implementer_roster: Vec::new(),
19642                judge_roster: Vec::new(),
19643                reviewer_roster: Vec::new(),
19644            },
19645            sem: Arc::new(Semaphore::new(1)),
19646            pause: Pause::new(),
19647            interrupt: Pause::new(),
19648        };
19649
19650        runner.merge().await.expect("merge");
19651
19652        assert!(
19653            runner.state.merge.is_none(),
19654            "an empty gate must never be read as a passing one: {:?}",
19655            runner.state.merge
19656        );
19657    }
19658
19659    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19660    /// commands configured and `merge.mode` is `none` (a review-only run).
19661    /// `gate` must still record a real attempt — zero commands, vacuously
19662    /// passed — rather than leaving `state.gate` empty in a way `merge`
19663    /// cannot tell apart from "never ran"; otherwise the run reaches
19664    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19665    #[tokio::test]
19666    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19667        let tmp = tempfile::tempdir().expect("tempdir");
19668        let repo = tmp.path().join("repo");
19669        std::fs::create_dir_all(&repo).unwrap();
19670        init_repo(&repo);
19671
19672        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19673        let config = Config::default();
19674
19675        let mut state = RunState::new(
19676            repo.clone(),
19677            "main".to_owned(),
19678            "deadbeef".to_owned(),
19679            "task".to_owned(),
19680            config,
19681        );
19682        state.candidates = vec![Candidate {
19683            index: 0,
19684            label: 'A',
19685            agent: "alpha".to_owned(),
19686            branch: "does-not-exist".to_owned(),
19687            worktree: repo.clone(),
19688            summary: String::new(),
19689            stat: String::new(),
19690            files: 0,
19691            commits: 0,
19692            empty: false,
19693            failed: None,
19694            verified_noop: None,
19695            duration_ms: 0,
19696            folded: false,
19697        }];
19698        state.tally = Some(Tally {
19699            first_choice: BTreeMap::from([('A', 1)]),
19700            borda: BTreeMap::new(),
19701            winner: 'A',
19702            rankings: 1,
19703            unanimous_initial: true,
19704            deliberated: false,
19705            changed_votes: 0,
19706            unanimous_final: true,
19707            tie_break: None,
19708            judges: 0,
19709            present: 0,
19710            quorum: 0,
19711            met_quorum: true,
19712            uncontested: Some("only candidate A produced a change".to_owned()),
19713        });
19714        state.reviews = vec![ReviewRound {
19715            round: 1,
19716            head: "deadbeef".to_owned(),
19717            verified_head: None,
19718            verified_at: None,
19719            reviews: Vec::new(),
19720            e2e: Vec::new(),
19721            fix: None,
19722            blocking: 0,
19723            answered: 0,
19724            expected: 0,
19725            clean: true,
19726            verify_retried: false,
19727            e2e_deferred: false,
19728            e2e_defer_reason: None,
19729            progressed: false,
19730            vote_split: false,
19731            reconsideration: Vec::new(),
19732            verdict: None,
19733        }];
19734
19735        let mut runner = Runner {
19736            state,
19737            roles: ResolvedRoles {
19738                implementers: Vec::new(),
19739                judges: Vec::new(),
19740                reviewers: Vec::new(),
19741                fixer: None,
19742                conductor: conductor(),
19743                implementer_roster: Vec::new(),
19744                judge_roster: Vec::new(),
19745                reviewer_roster: Vec::new(),
19746            },
19747            sem: Arc::new(Semaphore::new(1)),
19748            pause: Pause::new(),
19749            interrupt: Pause::new(),
19750        };
19751
19752        runner.gate().await.expect("gate");
19753        assert!(
19754            runner.state.gate_ran,
19755            "zero configured commands is still a real attempt, not an unrun gate"
19756        );
19757        assert!(runner.state.gate.is_empty());
19758        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19759        assert_ne!(
19760            runner.state.status,
19761            RunStatus::Blocked,
19762            "a gate with nothing to check must not read as failed"
19763        );
19764
19765        runner.merge().await.expect("merge");
19766        assert_eq!(
19767            runner.state.status,
19768            RunStatus::Ready,
19769            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19770        );
19771    }
19772
19773    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19774    /// `verify.gate` (so the e2e leg and the final gate never build against
19775    /// different directories). With zero `verify.gate` commands but a
19776    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19777    /// that lease before discovering it had nothing to run - so a repo with
19778    /// no gate commands could come back `resource_blocked` (and therefore
19779    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19780    /// cache, exactly the contention this run's own zero commands could
19781    /// never have touched. `gate` must recognise there is nothing to check
19782    /// before it ever asks for the lease.
19783    #[tokio::test]
19784    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19785        crate::run::pin_test_home();
19786        let home = crate::run::home();
19787
19788        let tmp = tempfile::tempdir().expect("tempdir");
19789        let repo = tmp.path().join("repo");
19790        std::fs::create_dir_all(&repo).unwrap();
19791        init_repo(&repo);
19792        // Unique to this test, so holding its lease cannot collide with
19793        // another test sharing the same process-wide `home`.
19794        let cache_dir = tmp.path().join("target");
19795
19796        let mut config = Config::default();
19797        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19798        // `verify.gate` stays empty (the default). Any accidental cache
19799        // acquisition must report contention immediately, regardless of how
19800        // long unrelated Git subprocesses take under parallel test load.
19801        config.graph.timeout_verify = Some(2);
19802        config.graph.timeout_resource_wait = 0;
19803
19804        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19805        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19806            .expect("no io error acquiring directly")
19807        {
19808            crate::cache::AcquireOutcome::Acquired(g) => g,
19809            crate::cache::AcquireOutcome::Busy(b) => {
19810                panic!("expected the direct acquire to win the lease first: {b:?}")
19811            }
19812        };
19813
19814        let mut state = RunState::new(
19815            repo.clone(),
19816            "main".to_owned(),
19817            "deadbeef".to_owned(),
19818            "task".to_owned(),
19819            config,
19820        );
19821        state.candidates = vec![Candidate {
19822            index: 0,
19823            label: 'A',
19824            agent: "alpha".to_owned(),
19825            branch: "does-not-exist".to_owned(),
19826            worktree: repo.clone(),
19827            summary: String::new(),
19828            stat: String::new(),
19829            files: 0,
19830            commits: 0,
19831            empty: false,
19832            failed: None,
19833            verified_noop: None,
19834            duration_ms: 0,
19835            folded: false,
19836        }];
19837        state.tally = Some(Tally {
19838            first_choice: BTreeMap::from([('A', 1)]),
19839            borda: BTreeMap::new(),
19840            winner: 'A',
19841            rankings: 1,
19842            unanimous_initial: true,
19843            deliberated: false,
19844            changed_votes: 0,
19845            unanimous_final: true,
19846            tie_break: None,
19847            judges: 0,
19848            present: 0,
19849            quorum: 0,
19850            met_quorum: true,
19851            uncontested: Some("only candidate A produced a change".to_owned()),
19852        });
19853        state.reviews = vec![ReviewRound {
19854            round: 1,
19855            head: "deadbeef".to_owned(),
19856            verified_head: None,
19857            verified_at: None,
19858            reviews: Vec::new(),
19859            e2e: Vec::new(),
19860            fix: None,
19861            blocking: 0,
19862            answered: 0,
19863            expected: 0,
19864            clean: true,
19865            verify_retried: false,
19866            e2e_deferred: false,
19867            e2e_defer_reason: None,
19868            progressed: false,
19869            vote_split: false,
19870            reconsideration: Vec::new(),
19871            verdict: None,
19872        }];
19873
19874        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19875        state.reviews[0].e2e = vec![CommandOutcome {
19876            command: state.config.verify.e2e[0].clone(),
19877            code: Some(0),
19878            output_tail: String::new(),
19879            duration_ms: 0,
19880            resource_blocked: false,
19881        }];
19882        let mut runner = Runner {
19883            state,
19884            roles: ResolvedRoles {
19885                implementers: Vec::new(),
19886                judges: Vec::new(),
19887                reviewers: Vec::new(),
19888                fixer: None,
19889                conductor: conductor(),
19890                implementer_roster: Vec::new(),
19891                judge_roster: Vec::new(),
19892                reviewer_roster: Vec::new(),
19893            },
19894            sem: Arc::new(Semaphore::new(1)),
19895            pause: Pause::new(),
19896            interrupt: Pause::new(),
19897        };
19898
19899        runner.gate().await.expect("gate");
19900        assert!(runner.state.resource_block.is_none());
19901        assert!(
19902            !runner
19903                .state
19904                .events
19905                .iter()
19906                .any(|event| { event.message.contains("waiting for the shared build cache") }),
19907            "a gate with nothing to run must never request the held cache lease"
19908        );
19909        assert!(
19910            runner.state.gate_ran,
19911            "zero commands is still a real, immediate attempt"
19912        );
19913        assert!(runner.state.gate.is_empty());
19914        assert_ne!(
19915            runner.state.status,
19916            RunStatus::Blocked,
19917            "must not read as resource-blocked on a lease it never asked for"
19918        );
19919    }
19920
19921    /// A separate process carries a live worker's environment throughout all
19922    /// verification entry points; the parallel harness never changes its env.
19923    #[cfg(unix)]
19924    #[tokio::test]
19925    async fn verification_is_standalone_inside_a_live_worker() {
19926        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19927        if std::env::var_os(CHILD).is_none() {
19928            let home = tempfile::tempdir().unwrap();
19929            let generation = crate::worker::pin_current(home.path()).unwrap();
19930            let output = std::process::Command::new(std::env::current_exe().unwrap())
19931                .args([
19932                    "--exact",
19933                    "graph::tests::verification_is_standalone_inside_a_live_worker",
19934                    "--nocapture",
19935                ])
19936                .env(CHILD, "1")
19937                .env("MAGI_HOME", home.path())
19938                .env("MAGI_WORKER_HOME", home.path())
19939                .env("MAGI_WORKER_ID", "abcdef")
19940                .env("MAGI_WORKER_GENERATION", &generation.id)
19941                .env("MAGI_WORKER_EXE", &generation.executable)
19942                .env("MAGI_RUN", "verification-parent")
19943                .env("MAGI_NODE", "gate")
19944                .env("MAGI_SEAT", "worker-fixture")
19945                .env("MAGI_PROCESS_OWNER", "fixture-owner")
19946                .env("MAGI_NO_AUTOUPDATE", "0")
19947                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19948                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19949                .env(
19950                    "VERIFY_EXPECT_TOOLCHAIN",
19951                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19952                )
19953                .env(
19954                    "VERIFY_EXPECT_CACHE",
19955                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19956                )
19957                .output()
19958                .unwrap();
19959            assert!(
19960                output.status.success()
19961                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19962                "{}\n{}",
19963                String::from_utf8_lossy(&output.stdout),
19964                String::from_utf8_lossy(&output.stderr)
19965            );
19966            return;
19967        }
19968        let home = crate::run::home();
19969        let directory = home.join("workers/jobs/abcdef");
19970        std::fs::create_dir_all(&directory).unwrap();
19971        let job = crate::worker::Job {
19972            id: "abcdef".into(),
19973            sequence: 1,
19974            kind: "task".into(),
19975            key: "task:verification".into(),
19976            payload: serde_json::json!({}),
19977            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19978            version: Some(env!("CARGO_PKG_VERSION").into()),
19979            status: crate::worker::Status::Running,
19980            result: None,
19981            pid: Some(std::process::id()),
19982            created_at: 0,
19983            finished_at: None,
19984        };
19985        std::fs::write(
19986            directory.join("job.json"),
19987            serde_json::to_vec(&job).unwrap(),
19988        )
19989        .unwrap();
19990        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19991        let assert_worker = || {
19992            assert_eq!(
19993                crate::worker::current_job(&home).unwrap().unwrap().id,
19994                job.id
19995            );
19996            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19997            assert_eq!(
19998                crate::worker::helper_executable(&home).unwrap(),
19999                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
20000            );
20001            assert!(
20002                crate::updater::Checker::new(&crate::config::Update {
20003                    mode: crate::config::UpdateMode::Notify,
20004                    interval: None,
20005                    auto_restart: false,
20006                })
20007                .is_none(),
20008                "the driver remains pinned, even while tests are standalone"
20009            );
20010        };
20011        assert_worker();
20012        let repo = home.join("repo");
20013        std::fs::create_dir_all(&repo).unwrap();
20014        init_repo(&repo);
20015        // Keep storage, seat/control context, containment and tool lookup.
20016        // No inherited values (or credentials) are printed on success.
20017        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20018            test -z "${MAGI_WORKER_GENERATION+x}" &&
20019            test -z "${MAGI_WORKER_HOME+x}" &&
20020            test -z "${MAGI_WORKER_EXE+x}" &&
20021            test "$MAGI_HOME/repo" -ef . &&
20022            test "$MAGI_RUN" = verification-parent &&
20023            test "$MAGI_NODE" = gate &&
20024            test "$MAGI_SEAT" = worker-fixture &&
20025            test -n "$MAGI_PROCESS_OWNER" &&
20026            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20027            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20028            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20029            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20030        // Negative control: an ordinary child inherits the contamination.
20031        assert!(
20032            !std::process::Command::new("sh")
20033                .args(["-c", probe])
20034                .current_dir(&repo)
20035                .status()
20036                .unwrap()
20037                .success()
20038        );
20039        let unrelated_home = tempfile::tempdir().unwrap();
20040        assert!(
20041            crate::worker::current_job(unrelated_home.path()).is_err(),
20042            "an inherited worker id cannot be resolved in a unit fixture's home"
20043        );
20044        let mut runner = runner_at(RunStatus::Gating);
20045        runner.state.repo = repo.clone();
20046        runner.state.config.disk.min_free_bytes = 0;
20047        runner.state.config.coordination = crate::config::test_coordination();
20048        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20049        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20050        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20051        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20052        let winner = Candidate {
20053            index: 0,
20054            label: 'A',
20055            agent: "fixture".into(),
20056            branch: "main".into(),
20057            worktree: repo.clone(),
20058            summary: String::new(),
20059            stat: String::new(),
20060            files: 0,
20061            commits: 0,
20062            empty: false,
20063            failed: None,
20064            verified_noop: None,
20065            duration_ms: 0,
20066            folded: false,
20067        };
20068        let (e2e, retried, pids) = run_e2e_with_retry(
20069            &mut runner.state,
20070            &["sh".into(), "-c".into()],
20071            &[format!("{probe} && printf initial")],
20072            &repo,
20073            Duration::from_secs(10),
20074            "initial",
20075        )
20076        .await;
20077        assert!(!retried && pids.is_empty());
20078        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20079        assert_eq!(e2e[0].output_tail, "initial");
20080        let gate = runner.run_gate(&winner).await.unwrap();
20081        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20082        // An interrupted repair must recheck e2e from saved state. Landing
20083        // reuses that state and must run both pre-gate and gate again.
20084        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20085            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20086            passed: false,
20087            outcomes: Vec::new(),
20088        });
20089        runner.state.save().unwrap();
20090        let mut resumed = RunState::load(&runner.state.id).unwrap();
20091        let wait = crate::repo_lock::WaitPolicy::default();
20092        assert!(matches!(
20093            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20094                .await
20095                .unwrap(),
20096            GateFix::Retry
20097        ));
20098        let saved = RunState::load(&resumed.id).unwrap();
20099        let repair = saved.gate_repair_e2e.unwrap();
20100        assert!(repair.passed);
20101        assert_eq!(repair.outcomes.len(), 1);
20102        assert!(repair.outcomes[0].ok());
20103        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20104            .await
20105            .unwrap();
20106        assert_eq!(landing.len(), 1);
20107        assert!(landing[0].ok(), "{landing:?}");
20108        assert_eq!(
20109            std::fs::read_to_string(repo.join("phases")).unwrap(),
20110            "pre\ngate\ne2e\npre\ngate\n"
20111        );
20112        assert_worker();
20113    }
20114
20115    #[cfg(unix)]
20116    #[tokio::test]
20117    async fn verification_keeps_the_panic_and_the_full_output() {
20118        crate::run::pin_test_home();
20119        let tmp = tempfile::tempdir().unwrap();
20120        let mut config = Config::default();
20121        config.disk.min_free_bytes = 0;
20122        let mut state = RunState::new(
20123            tmp.path().to_owned(),
20124            "main".into(),
20125            "base".into(),
20126            "verification evidence".into(),
20127            config,
20128        );
20129        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20130                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20131                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20132        let mut names = Vec::new();
20133        for _ in 0..2 {
20134            let (outcomes, _) = run_commands(
20135                &mut state,
20136                "gate",
20137                "gate",
20138                0,
20139                &["sh".into(), "-c".into()],
20140                &[script.into()],
20141                tmp.path(),
20142                Duration::from_secs(60),
20143            )
20144            .await;
20145            let tail = &outcomes[0].output_tail;
20146            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20147            assert!(tail.contains("boom assertion"));
20148            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20149            let line = tail.lines().next().unwrap();
20150            names.push(line.to_owned());
20151        }
20152        assert_ne!(names[0], names[1], "the second run must not overwrite");
20153        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20154        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20155        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20156
20157        // Storage failures are evidence too; preserve the command's verdict.
20158        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20159        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20160        let (outcomes, _) = run_commands(
20161            &mut state,
20162            "gate",
20163            "gate",
20164            0,
20165            &["sh".into(), "-c".into()],
20166            &["echo failure-output; exit 7".into()],
20167            tmp.path(),
20168            Duration::from_secs(10),
20169        )
20170        .await;
20171        assert_eq!(outcomes[0].code, Some(7));
20172        assert!(state.events.iter().any(|e| {
20173            e.node == "gate" && e.message.contains("could not save the full gate output:")
20174        }));
20175    }
20176
20177    #[cfg(unix)]
20178    #[tokio::test]
20179    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20180        crate::run::pin_test_home();
20181        let tmp = tempfile::tempdir().unwrap();
20182        let mut config = Config::default();
20183        config.disk.min_free_bytes = 0;
20184        let mut state = RunState::new(
20185            tmp.path().to_owned(),
20186            "main".into(),
20187            "base".into(),
20188            "verification evidence".into(),
20189            config,
20190        );
20191        let (outcomes, pids) = run_commands(
20192            &mut state,
20193            "gate",
20194            "gate",
20195            1,
20196            &["sh".into(), "-c".into()],
20197            &[
20198                "printf before-timeout; exec sleep 30".into(),
20199                "kill -TERM $$".into(),
20200                "exit 7".into(),
20201                "printf success".into(),
20202            ],
20203            tmp.path(),
20204            Duration::from_secs(1),
20205        )
20206        .await;
20207        wait_for_timed_out_children_to_die(&pids).await;
20208        assert_eq!(
20209            pids.len(),
20210            1,
20211            "unleased timeout retains its death-check PID"
20212        );
20213        assert_eq!(outcomes.len(), 4);
20214        assert_eq!(outcomes[0].code, None);
20215        assert!(outcomes[0].output_tail.contains("before-timeout"));
20216        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20217        assert_eq!(outcomes[1].code, None);
20218        assert!(outcomes[1].output_tail.contains("command terminated:"));
20219        assert!(!outcomes[1].output_tail.contains("timed out"));
20220        assert_eq!(outcomes[2].code, Some(7));
20221        assert!(!outcomes[2].output_tail.contains("timed out"));
20222        assert_eq!(outcomes[3].code, Some(0));
20223        assert_eq!(outcomes[3].output_tail, "success");
20224        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20225    }
20226
20227    /// The addendum's second gap: a `verify.gate` command running for real
20228    /// wall-clock time had nothing at all to show for it in `active` before
20229    /// `run_commands` learned to record it — a run could sit in `Gating` for
20230    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20231    /// what was actually happening. Proven with a genuinely still-running
20232    /// command, not just a before/after check on the final state: a poller
20233    /// task reads the same `run.json` `gate()` is writing, the same way the
20234    /// phone or `magi show` would, while the shell command is still blocked
20235    /// on its own release marker.
20236    #[tokio::test]
20237    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20238        crate::run::pin_test_home();
20239
20240        let tmp = tempfile::tempdir().expect("tempdir");
20241        let repo = tmp.path().join("repo");
20242        std::fs::create_dir_all(&repo).unwrap();
20243        init_repo(&repo);
20244
20245        let mut config = Config::default();
20246        config.disk.min_free_bytes = 0;
20247        config.verify.gate = vec![
20248            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20249             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20250                .to_owned(),
20251        ];
20252
20253        let mut state = RunState::new(
20254            repo.clone(),
20255            "main".to_owned(),
20256            "deadbeef".to_owned(),
20257            "task".to_owned(),
20258            config,
20259        );
20260        let run_id = state.id.clone();
20261        state.candidates = vec![Candidate {
20262            index: 0,
20263            label: 'A',
20264            agent: "alpha".to_owned(),
20265            branch: "does-not-exist".to_owned(),
20266            worktree: repo.clone(),
20267            summary: String::new(),
20268            stat: String::new(),
20269            files: 0,
20270            commits: 0,
20271            empty: false,
20272            failed: None,
20273            verified_noop: None,
20274            duration_ms: 0,
20275            folded: false,
20276        }];
20277        state.tally = Some(Tally {
20278            first_choice: BTreeMap::from([('A', 1)]),
20279            borda: BTreeMap::new(),
20280            winner: 'A',
20281            rankings: 1,
20282            unanimous_initial: true,
20283            deliberated: false,
20284            changed_votes: 0,
20285            unanimous_final: true,
20286            tie_break: None,
20287            judges: 0,
20288            present: 0,
20289            quorum: 0,
20290            met_quorum: true,
20291            uncontested: Some("only candidate A produced a change".to_owned()),
20292        });
20293        state.reviews = vec![ReviewRound {
20294            round: 1,
20295            head: "deadbeef".to_owned(),
20296            verified_head: None,
20297            verified_at: None,
20298            reviews: Vec::new(),
20299            e2e: Vec::new(),
20300            fix: None,
20301            blocking: 0,
20302            answered: 0,
20303            expected: 0,
20304            clean: true,
20305            verify_retried: false,
20306            e2e_deferred: false,
20307            e2e_defer_reason: None,
20308            progressed: false,
20309            vote_split: false,
20310            reconsideration: Vec::new(),
20311            verdict: None,
20312        }];
20313
20314        let mut runner = Runner {
20315            state,
20316            roles: ResolvedRoles {
20317                implementers: Vec::new(),
20318                judges: Vec::new(),
20319                reviewers: Vec::new(),
20320                fixer: None,
20321                conductor: conductor(),
20322                implementer_roster: Vec::new(),
20323                judge_roster: Vec::new(),
20324                reviewer_roster: Vec::new(),
20325            },
20326            sem: Arc::new(Semaphore::new(1)),
20327            pause: Pause::new(),
20328            interrupt: Pause::new(),
20329        };
20330
20331        let started_marker = repo.join("started.marker");
20332        let release_marker = repo.join("release.marker");
20333        let poller = tokio::spawn(async move {
20334            // Bounded so a regression that never records the task entry
20335            // fails this test in seconds instead of hanging the suite —
20336            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20337            // does_not_cut_it_short` uses for the same reason.
20338            for _ in 0..100 {
20339                if started_marker.exists()
20340                    && let Ok(s) = crate::run::RunState::load(&run_id)
20341                    && let Some(a) = s.active.get("gate")
20342                {
20343                    std::fs::write(&release_marker, b"go").expect("release marker");
20344                    return Some(a.clone());
20345                }
20346                tokio::time::sleep(Duration::from_millis(50)).await;
20347            }
20348            None
20349        });
20350
20351        runner.gate().await.expect("gate");
20352        let captured = poller.await.expect("poller task");
20353        let captured = captured.expect(
20354            "the poller never saw a `gate` task entry in run.json while the command was \
20355             still blocked on its own release marker",
20356        );
20357
20358        assert_eq!(captured.task.as_deref(), Some("gate"));
20359        assert_eq!(captured.node, "gate");
20360        assert_eq!(captured.index, Some(1));
20361        assert_eq!(captured.total, Some(1));
20362        assert!(
20363            captured
20364                .command
20365                .as_deref()
20366                .is_some_and(|c| c.contains("started.marker")),
20367            "{captured:?}"
20368        );
20369
20370        assert!(
20371            runner.state.active.is_empty(),
20372            "the entry must be cleared once the command actually finished: {:?}",
20373            runner.state.active
20374        );
20375        assert!(runner.state.gate_ran);
20376        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20377    }
20378
20379    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20380    /// record for `land`; one with only a Minor, or no reject, leaves none.
20381    #[tokio::test]
20382    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20383        use crate::verdict::{Finding, ReviewVote, Severity};
20384        crate::run::pin_test_home();
20385        let tmp = tempfile::tempdir().expect("tempdir");
20386        let repo = tmp.path().join("repo");
20387        std::fs::create_dir_all(&repo).unwrap();
20388        init_repo(&repo);
20389
20390        for (severity, vote, expect) in [
20391            (Severity::Major, ReviewVote::Reject, true),
20392            (Severity::Minor, ReviewVote::Reject, false),
20393            (Severity::Major, ReviewVote::Approve, false),
20394        ] {
20395            let mut round = review_round(false, 1, 1, 1, false, true);
20396            round.reviews = vec![ReviewRecord {
20397                reviewer: 1,
20398                agent: "alpha".to_owned(),
20399                summary: String::new(),
20400                findings: vec![Finding {
20401                    id: "R1-1-1".to_owned(),
20402                    severity,
20403                    file: None,
20404                    line: None,
20405                    title: "t".to_owned(),
20406                    detail: String::new(),
20407                }],
20408                vote: Some(vote),
20409                failed: None,
20410                duration_ms: 0,
20411                attempts: 0,
20412            }];
20413            let mut state = RunState::new(
20414                repo.clone(),
20415                "main".to_owned(),
20416                "deadbeef".to_owned(),
20417                "task".to_owned(),
20418                Config::default(),
20419            );
20420            state.reviews = vec![round];
20421            let mut runner = Runner {
20422                state,
20423                roles: ResolvedRoles {
20424                    implementers: Vec::new(),
20425                    judges: Vec::new(),
20426                    reviewers: Vec::new(),
20427                    fixer: None,
20428                    conductor: conductor(),
20429                    implementer_roster: Vec::new(),
20430                    judge_roster: Vec::new(),
20431                    reviewer_roster: Vec::new(),
20432                },
20433                sem: Arc::new(Semaphore::new(1)),
20434                pause: Pause::new(),
20435                interrupt: Pause::new(),
20436            };
20437            let shell = runner.state.config.shell();
20438            runner
20439                .stop_reviewing("round budget spent", &shell, &repo)
20440                .await
20441                .expect("stop_reviewing");
20442            assert_eq!(runner.state.status, RunStatus::Gating);
20443            assert_eq!(
20444                runner.state.contested_handoff.is_some(),
20445                expect,
20446                "{severity:?} + {vote:?}"
20447            );
20448        }
20449    }
20450
20451    /// The shape the incident this whole fix responds to actually had: the
20452    /// round budget spent, the last round's own e2e blocked on the shared
20453    /// build cache (held here by a live pid — this test process — exactly
20454    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20455    /// without forking a process). `stop_reviewing` must retry it — not
20456    /// silently leave the round looking untouched (the catch-up-only half of
20457    /// the bug), and not read the contention as a red `e2e` and block the
20458    /// run on it (the other half). Called directly, the same way
20459    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20460    /// above exercises `gate`, so this never needs a real cargo build to
20461    /// reach: the lease is never released, so `with_cache_lease` never gets
20462    /// past acquiring it into anything that would need a real workspace.
20463    #[tokio::test]
20464    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20465        crate::run::pin_test_home();
20466        let home = crate::run::home();
20467
20468        let tmp = tempfile::tempdir().expect("tempdir");
20469        let repo = tmp.path().join("repo");
20470        std::fs::create_dir_all(&repo).unwrap();
20471        init_repo(&repo);
20472        let head = crate::git::rev_parse(&repo, "HEAD")
20473            .await
20474            .expect("rev-parse");
20475        // Unique to this test, so holding its lease cannot collide with
20476        // another test sharing the same process-wide `home`.
20477        let cache_dir = tmp.path().join("target");
20478
20479        let mut config = Config::default();
20480        config.verify.e2e = vec![format!(
20481            "CARGO_TARGET_DIR='{}' test -f README.md",
20482            cache_dir.display()
20483        )];
20484        config.graph.review_rounds = 1;
20485        // Bounded so a regression that does start waiting fails the test in
20486        // seconds, not hangs it.
20487        config.graph.timeout_verify = Some(2);
20488        config.graph.timeout_resource_wait = 2;
20489
20490        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20491        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20492            .expect("no io error acquiring directly")
20493        {
20494            crate::cache::AcquireOutcome::Acquired(g) => g,
20495            crate::cache::AcquireOutcome::Busy(b) => {
20496                panic!("expected the direct acquire to win the lease first: {b:?}")
20497            }
20498        };
20499
20500        let mut state = RunState::new(
20501            repo.clone(),
20502            "main".to_owned(),
20503            head.clone(),
20504            "task".to_owned(),
20505            config,
20506        );
20507        state.candidates = vec![Candidate {
20508            index: 0,
20509            label: 'A',
20510            agent: "alpha".to_owned(),
20511            branch: "does-not-exist".to_owned(),
20512            worktree: repo.clone(),
20513            summary: String::new(),
20514            stat: String::new(),
20515            files: 0,
20516            commits: 0,
20517            empty: false,
20518            failed: None,
20519            verified_noop: None,
20520            duration_ms: 0,
20521            folded: false,
20522        }];
20523        state.tally = Some(Tally {
20524            first_choice: BTreeMap::from([('A', 1)]),
20525            borda: BTreeMap::new(),
20526            winner: 'A',
20527            rankings: 1,
20528            unanimous_initial: true,
20529            deliberated: false,
20530            changed_votes: 0,
20531            unanimous_final: true,
20532            tie_break: None,
20533            judges: 0,
20534            present: 0,
20535            quorum: 0,
20536            met_quorum: true,
20537            uncontested: Some("only candidate A produced a change".to_owned()),
20538        });
20539        // The round budget's last round, deferred: `needs_catchup_run`'s
20540        // other trigger. `stop_reviewing`'s retry machinery must treat this
20541        // exactly like a resource-blocked attempt once it actually runs.
20542        state.reviews = vec![ReviewRound {
20543            round: 1,
20544            head: head.clone(),
20545            verified_head: None,
20546            verified_at: None,
20547            reviews: Vec::new(),
20548            e2e: Vec::new(),
20549            fix: None,
20550            blocking: 1,
20551            answered: 1,
20552            expected: 1,
20553            clean: false,
20554            verify_retried: false,
20555            e2e_deferred: true,
20556            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20557            progressed: false,
20558            vote_split: false,
20559            reconsideration: Vec::new(),
20560            verdict: None,
20561        }];
20562
20563        let mut runner = Runner {
20564            state,
20565            roles: ResolvedRoles {
20566                implementers: Vec::new(),
20567                judges: Vec::new(),
20568                reviewers: Vec::new(),
20569                fixer: None,
20570                conductor: conductor(),
20571                implementer_roster: Vec::new(),
20572                judge_roster: Vec::new(),
20573                reviewer_roster: Vec::new(),
20574            },
20575            sem: Arc::new(Semaphore::new(1)),
20576            pause: Pause::new(),
20577            interrupt: Pause::new(),
20578        };
20579
20580        let shell = runner.state.config.shell();
20581        runner
20582            .stop_reviewing("round budget spent", &shell, &repo)
20583            .await
20584            .expect("stop_reviewing");
20585
20586        let last = runner.state.reviews.last().expect("round record");
20587        assert_eq!(
20588            last.e2e_status(),
20589            E2eStatus::ResourceBlocked,
20590            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20591             failed: {last:?}"
20592        );
20593        assert_eq!(
20594            last.verified_head.as_deref(),
20595            Some(head.as_str()),
20596            "which commit this attempt targeted is known even though nothing finished checking \
20597             it"
20598        );
20599        let first_attempt_at = last
20600            .verified_at
20601            .expect("when this attempt ran is known too");
20602        assert_ne!(
20603            runner.state.status,
20604            RunStatus::Blocked,
20605            "contention is evidence about the machine, not the patch — it must not settle the \
20606             run as blocked: {:?}",
20607            runner.state.status
20608        );
20609        assert!(
20610            !runner
20611                .state
20612                .events
20613                .iter()
20614                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20615            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20616            runner.state.events
20617        );
20618
20619        // The cache is still held: a later reentry must retry the same
20620        // round's verification again — not leave it looking exactly as
20621        // untouched as the first blocked attempt, which is indistinguishable
20622        // from never having tried again at all.
20623        runner
20624            .stop_reviewing("round budget spent", &shell, &repo)
20625            .await
20626            .expect("stop_reviewing retry");
20627        assert_eq!(
20628            runner.state.reviews.len(),
20629            1,
20630            "no new round was started: {:?}",
20631            runner.state.reviews
20632        );
20633        let last = runner.state.reviews.last().expect("round record");
20634        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20635        assert!(
20636            last.verified_at.expect("still known") > first_attempt_at,
20637            "a second reentry must be a fresh attempt, not a stale copy of the first"
20638        );
20639        assert_ne!(runner.state.status, RunStatus::Blocked);
20640
20641        held.release();
20642    }
20643
20644    #[cfg(unix)]
20645    #[tokio::test]
20646    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20647        ask_test_home();
20648        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20649            let tmp = tempfile::tempdir().unwrap();
20650            let repo = tmp.path().join("repo");
20651            std::fs::create_dir(&repo).unwrap();
20652            init_repo(&repo);
20653            let run_git = |args: &[&str]| {
20654                let out = std::process::Command::new("git")
20655                    .args(args)
20656                    .current_dir(&repo)
20657                    .output()
20658                    .unwrap();
20659                assert!(
20660                    out.status.success(),
20661                    "{args:?}: {}",
20662                    String::from_utf8_lossy(&out.stderr)
20663                );
20664            };
20665            let origin = tmp.path().join("origin.git");
20666            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20667            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20668            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20669            run_git(&["add", "spec.rs"]);
20670            run_git(&["commit", "-m", "initial spec"]);
20671            run_git(&["push", "origin", "main"]);
20672            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20673            run_git(&["checkout", "-b", "fix/review"]);
20674            std::fs::write(
20675                repo.join("check.rs"),
20676                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20677            )
20678            .unwrap();
20679            run_git(&["add", "check.rs"]);
20680            run_git(&["commit", "-m", "use spec"]);
20681            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20682            run_git(&["checkout", "main"]);
20683            std::fs::write(
20684                repo.join("spec.rs"),
20685                "struct AgentSpec { max_turn_secs: u64 }\n",
20686            )
20687            .unwrap();
20688            run_git(&["add", "spec.rs"]);
20689            run_git(&["commit", "-m", "extend spec"]);
20690            run_git(&["push", "origin", "main"]);
20691            run_git(&["checkout", "fix/review"]);
20692
20693            let mut runner = runner_at(RunStatus::Reviewing);
20694            runner.state.repo = repo.clone();
20695            runner.state.base_commit = base;
20696            runner.state.pr_head = Some("fix/review".into());
20697            runner.state.config.graph.review_rounds = max_rounds;
20698            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20699            runner.state.config.graph.sessions = false;
20700            runner.state.config.disk.min_free_bytes = 0;
20701            runner.state.config.verify.e2e =
20702                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20703            runner.state.candidates = vec![Candidate {
20704                index: 0,
20705                label: 'A',
20706                agent: "fixture".into(),
20707                branch: "fix/review".into(),
20708                worktree: repo.clone(),
20709                summary: String::new(),
20710                stat: String::new(),
20711                files: 1,
20712                commits: 1,
20713                empty: false,
20714                failed: None,
20715                verified_noop: None,
20716                duration_ms: 0,
20717                folded: false,
20718            }];
20719            runner.state.tally = Some(Tally {
20720                first_choice: BTreeMap::new(),
20721                borda: BTreeMap::new(),
20722                winner: 'A',
20723                rankings: 0,
20724                unanimous_initial: true,
20725                deliberated: false,
20726                changed_votes: 0,
20727                unanimous_final: true,
20728                tie_break: None,
20729                judges: 0,
20730                present: 0,
20731                quorum: 0,
20732                met_quorum: true,
20733                uncontested: Some("review-only".into()),
20734            });
20735            let mut last = review_round(false, 0, 1, 1, false, false);
20736            last.head = reviewed.clone();
20737            last.e2e_deferred = true;
20738            last.e2e[0].resource_blocked = true;
20739            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20740            if max_rounds == 1 {
20741                last.e2e.clear();
20742            }
20743            runner.state.reviews = vec![last];
20744            let calls = tmp.path().join("fix-calls");
20745            let mut fixer = spec("repair");
20746            fixer
20747                .env
20748                .insert("FIX_CALLS".into(), calls.display().to_string());
20749            fixer.command = vec!["sh".into(), "-c".into(), r#"
20750                echo fix >> "$FIX_CALLS"
20751                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20752                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20753                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20754                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20755            "#.into()];
20756            let mut reviewer = spec("reviewer");
20757            reviewer.command = vec![
20758                "sh".into(),
20759                "-c".into(),
20760                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20761                    .into(),
20762            ];
20763            runner.roles.fixer = Some(vec![fixer]);
20764            runner.roles.reviewers = vec![reviewer];
20765
20766            runner.sync_to_base().await.unwrap();
20767            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20768            assert_ne!(rebased, reviewed);
20769            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20770            if reentry {
20771                // Simulate a crash after catch-up evidence was saved but before
20772                // dispatching the fixer. No dedicated pending-repair field exists.
20773                let shell = runner.state.config.shell();
20774                runner
20775                    .stop_unresolved_review("deferred verification", &shell, &repo)
20776                    .await
20777                    .unwrap();
20778                assert!(runner.catchup_repairable());
20779                assert!(!calls.exists());
20780                runner.state = RunState::load(&runner.state.id).unwrap();
20781                assert!(runner.catchup_repairable());
20782            }
20783            runner.review_loop().await.unwrap();
20784            let first = &runner.state.reviews[0];
20785            assert_eq!(first.head, reviewed);
20786            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20787            assert!(
20788                first.verify_retried,
20789                "a reproduced build failure gets exactly one retry: {:?}",
20790                first.e2e
20791            );
20792            assert!(first.e2e[0].output_tail.contains("E0063"));
20793            assert!(!runner.catchup_repairable());
20794            if max_rounds == 2 {
20795                assert_eq!(
20796                    runner.state.status,
20797                    RunStatus::Gating,
20798                    "{}",
20799                    crate::report::run(&runner.state)
20800                );
20801                assert_eq!(runner.state.reviews.len(), 2);
20802                assert!(first.fix.as_ref().unwrap().committed);
20803                assert!(runner.state.reviews[1].clean);
20804                assert_eq!(
20805                    runner
20806                        .state
20807                        .reviews
20808                        .iter()
20809                        .filter(|r| r.fix.is_some())
20810                        .count(),
20811                    1
20812                );
20813                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20814                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20815            } else {
20816                assert_eq!(runner.state.status, RunStatus::Blocked);
20817                assert!(first.fix.is_none());
20818                assert!(!calls.exists());
20819                let recovery = crate::recovery::assess(
20820                    "prior",
20821                    Some(&runner.state),
20822                    &runner.landing_base(),
20823                    crate::recovery::Relation::Diverged,
20824                );
20825                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20826                let reasons = recovery.reasons.join("\n");
20827                assert!(
20828                    reasons.contains(&reviewed) && reasons.contains(&rebased),
20829                    "{reasons}"
20830                );
20831                assert!(
20832                    reasons.contains("base sync to")
20833                        && reasons.contains("additional review is required"),
20834                    "{reasons}"
20835                );
20836            }
20837        }
20838    }
20839
20840    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
20841    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
20842    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
20843    /// own top-of-function fast path (`review_conclusion`) correctly reads
20844    /// this shape as `None` rather than guessing `Blocked`, and the loop's
20845    /// own `for` range is empty once the round budget is spent, so
20846    /// `review_loop` must retry the check itself rather than silently doing
20847    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
20848    /// above exercises directly, but through `review_loop`'s own entry point
20849    /// this time, proving the wiring between the two rather than just the
20850    /// retry logic in isolation.
20851    #[tokio::test]
20852    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20853        crate::run::pin_test_home();
20854        let home = crate::run::home();
20855
20856        let tmp = tempfile::tempdir().expect("tempdir");
20857        let repo = tmp.path().join("repo");
20858        std::fs::create_dir_all(&repo).unwrap();
20859        init_repo(&repo);
20860        let head = crate::git::rev_parse(&repo, "HEAD")
20861            .await
20862            .expect("rev-parse");
20863        let cache_dir = tmp.path().join("target");
20864
20865        let mut config = Config::default();
20866        config.verify.e2e = vec![format!(
20867            "CARGO_TARGET_DIR='{}' test -f README.md",
20868            cache_dir.display()
20869        )];
20870        config.graph.review_rounds = 1;
20871        config.graph.timeout_verify = Some(2);
20872        config.graph.timeout_resource_wait = 2;
20873
20874        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20875        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20876            .expect("no io error acquiring directly")
20877        {
20878            crate::cache::AcquireOutcome::Acquired(g) => g,
20879            crate::cache::AcquireOutcome::Busy(b) => {
20880                panic!("expected the direct acquire to win the lease first: {b:?}")
20881            }
20882        };
20883
20884        let mut state = RunState::new(
20885            repo.clone(),
20886            "main".to_owned(),
20887            head.clone(),
20888            "task".to_owned(),
20889            config,
20890        );
20891        state.candidates = vec![Candidate {
20892            index: 0,
20893            label: 'A',
20894            agent: "alpha".to_owned(),
20895            branch: "does-not-exist".to_owned(),
20896            worktree: repo.clone(),
20897            summary: String::new(),
20898            stat: String::new(),
20899            files: 0,
20900            commits: 0,
20901            empty: false,
20902            failed: None,
20903            verified_noop: None,
20904            duration_ms: 0,
20905            folded: false,
20906        }];
20907        state.tally = Some(Tally {
20908            first_choice: BTreeMap::from([('A', 1)]),
20909            borda: BTreeMap::new(),
20910            winner: 'A',
20911            rankings: 1,
20912            unanimous_initial: true,
20913            deliberated: false,
20914            changed_votes: 0,
20915            unanimous_final: true,
20916            tie_break: None,
20917            judges: 0,
20918            present: 0,
20919            quorum: 0,
20920            met_quorum: true,
20921            uncontested: Some("only candidate A produced a change".to_owned()),
20922        });
20923        // The exact shape a prior process's `stop_reviewing` would have left
20924        // on disk: the round budget's last round, a real attempt already
20925        // made and already resource-blocked.
20926        state.reviews = vec![ReviewRound {
20927            round: 1,
20928            head: head.clone(),
20929            verified_head: Some(head.clone()),
20930            verified_at: Some(jiff::Timestamp::now()),
20931            reviews: Vec::new(),
20932            e2e: vec![CommandOutcome {
20933                command: format!(
20934                    "CARGO_TARGET_DIR='{}' test -f README.md",
20935                    cache_dir.display()
20936                ),
20937                code: None,
20938                output_tail: "waiting for the shared build cache".to_owned(),
20939                duration_ms: 0,
20940                resource_blocked: true,
20941            }],
20942            fix: None,
20943            blocking: 1,
20944            answered: 1,
20945            expected: 1,
20946            clean: false,
20947            verify_retried: false,
20948            e2e_deferred: false,
20949            e2e_defer_reason: None,
20950            progressed: false,
20951            vote_split: false,
20952            reconsideration: Vec::new(),
20953            verdict: None,
20954        }];
20955
20956        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20957        let mut runner = Runner {
20958            state,
20959            roles: ResolvedRoles {
20960                implementers: Vec::new(),
20961                judges: Vec::new(),
20962                reviewers: Vec::new(),
20963                fixer: None,
20964                conductor: conductor(),
20965                implementer_roster: Vec::new(),
20966                judge_roster: Vec::new(),
20967                reviewer_roster: Vec::new(),
20968            },
20969            sem: Arc::new(Semaphore::new(1)),
20970            pause: Pause::new(),
20971            interrupt: Pause::new(),
20972        };
20973
20974        // The lease is still held throughout, so this reentry's own retry is
20975        // also contended — proving `review_loop` actually tried again (not
20976        // that it happened to succeed) is what the timestamp comparison
20977        // below is for.
20978        runner.review_loop().await.expect("review_loop");
20979
20980        assert_eq!(
20981            runner.state.reviews.len(),
20982            1,
20983            "no new round was started on top of the unresolved one: {:?}",
20984            runner.state.reviews
20985        );
20986        let last = &runner.state.reviews[0];
20987        assert_eq!(
20988            last.e2e_status(),
20989            E2eStatus::ResourceBlocked,
20990            "still contended: {last:?}"
20991        );
20992        assert!(
20993            last.verified_at.expect("still known") > first_attempt_at,
20994            "review_loop must have actually retried the check, not left it exactly as found"
20995        );
20996        assert_ne!(
20997            runner.state.status,
20998            RunStatus::Blocked,
20999            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
21000            runner.state.status
21001        );
21002
21003        held.release();
21004    }
21005
21006    #[tokio::test]
21007    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21008        crate::run::pin_test_home();
21009        for status in [RunStatus::Blocked, RunStatus::Gating] {
21010            let mut runner = runner_at(status);
21011            runner.state.config.graph.land = true;
21012            runner.state.gate_ran = true;
21013            runner.state.merge = Some(MergeOutcome {
21014                mode: MergeMode::Pr,
21015                ok: false,
21016                detail: "the fixer produced no commit".into(),
21017                empty: false,
21018            });
21019            let url = "https://example.invalid/o/r/pull/1";
21020            runner.state.pr = Some(crate::run::PrRecord {
21021                url: url.into(),
21022                number: 1,
21023                state: "open".into(),
21024                checks: "red".into(),
21025                round: 1,
21026                rounds: 1,
21027                red_at_merge: Vec::new(),
21028            });
21029            runner.state.save().unwrap();
21030            // The nonexistent repository makes observation fail before a forge
21031            // command can run. We still must reach land, rather than silently
21032            // return from the graph with its old gate and merge records.
21033            runner.execute_graph().await.unwrap();
21034            assert_eq!(runner.state.status, RunStatus::Blocked);
21035            assert!(runner.state.land_sync_pending);
21036            assert!(runner.state.events.iter().any(|event|
21037                event.node == "land" && event.message == format!("watching {url}")));
21038            assert_eq!(
21039                runner.state.merge.as_ref().unwrap().detail,
21040                "the fixer produced no commit"
21041            );
21042        }
21043    }
21044
21045    #[tokio::test]
21046    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21047        crate::run::pin_test_home();
21048        let tmp = tempfile::tempdir().expect("tempdir");
21049        let repo = tmp.path().join("repo");
21050        std::fs::create_dir_all(&repo).unwrap();
21051        init_repo(&repo);
21052
21053        let mut config = Config::default();
21054        config.merge.mode = MergeMode::Pr;
21055        config.graph.land = true;
21056        config.graph.land_approval = false;
21057
21058        let mut state = RunState::new(
21059            repo.clone(),
21060            "main".to_owned(),
21061            "deadbeef".to_owned(),
21062            "task".to_owned(),
21063            config,
21064        );
21065        state.candidates = vec![Candidate {
21066            index: 0,
21067            label: 'A',
21068            agent: "alpha".to_owned(),
21069            branch: "does-not-exist".to_owned(),
21070            worktree: repo.clone(),
21071            summary: String::new(),
21072            stat: String::new(),
21073            files: 0,
21074            commits: 0,
21075            empty: false,
21076            failed: None,
21077            verified_noop: None,
21078            duration_ms: 0,
21079            folded: false,
21080        }];
21081        state.tally = Some(Tally {
21082            first_choice: BTreeMap::from([('A', 1)]),
21083            borda: BTreeMap::new(),
21084            winner: 'A',
21085            rankings: 1,
21086            unanimous_initial: true,
21087            deliberated: false,
21088            changed_votes: 0,
21089            unanimous_final: true,
21090            tie_break: None,
21091            judges: 0,
21092            present: 0,
21093            quorum: 0,
21094            met_quorum: true,
21095            uncontested: Some("only candidate A produced a change".to_owned()),
21096        });
21097        state.reviews = vec![ReviewRound {
21098            round: 1,
21099            head: "deadbeef".to_owned(),
21100            verified_head: None,
21101            verified_at: None,
21102            reviews: Vec::new(),
21103            e2e: Vec::new(),
21104            fix: None,
21105            blocking: 0,
21106            answered: 0,
21107            expected: 0,
21108            clean: true,
21109            verify_retried: false,
21110            e2e_deferred: false,
21111            e2e_defer_reason: None,
21112            progressed: false,
21113            vote_split: false,
21114            reconsideration: Vec::new(),
21115            verdict: None,
21116        }];
21117        state.gate = vec![CommandOutcome {
21118            command: "test".to_owned(),
21119            code: Some(0),
21120            output_tail: String::new(),
21121            duration_ms: 0,
21122            resource_blocked: false,
21123        }];
21124        state.gate_ran = true;
21125        // A first pass through `merge` already pushed and opened this pull
21126        // request; `status` is `Landing` because a previous call into `land`
21127        // parked or was interrupted before it reached a terminal outcome.
21128        state.status = RunStatus::Landing;
21129        state.merge = Some(MergeOutcome {
21130            mode: MergeMode::Pr,
21131            ok: true,
21132            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21133            empty: false,
21134        });
21135
21136        // The Landing-resume shortcut calls `run_land` directly rather than
21137        // through `merge`, which is exactly the call site that used to skip
21138        // `settle_questions` - see the fixture below.
21139        ask_test_home();
21140        // A real resume loads a persisted run; driver ownership is checked
21141        // against that record before the Landing shortcut is entered.
21142        state.save().expect("save landing run");
21143        let store = ask::Questions::open();
21144        let q = ask_open_question(&store, &state.id);
21145
21146        let mut runner = Runner {
21147            state,
21148            roles: ResolvedRoles {
21149                implementers: Vec::new(),
21150                judges: Vec::new(),
21151                reviewers: Vec::new(),
21152                fixer: None,
21153                conductor: conductor(),
21154                implementer_roster: Vec::new(),
21155                judge_roster: Vec::new(),
21156                reviewer_roster: Vec::new(),
21157            },
21158            sem: Arc::new(Semaphore::new(1)),
21159            pause: Pause::new(),
21160            interrupt: Pause::new(),
21161        };
21162
21163        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21164        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21165        // exactly because `review_loop` would otherwise clobber the marker
21166        // first.
21167        runner.execute().await.expect("execute");
21168
21169        assert_eq!(
21170            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21171            Some("https://example.invalid/x/y/pull/1"),
21172            "reentry must not push again or open a second pull request over the \
21173             one `land` is already watching"
21174        );
21175        assert_ne!(
21176            runner.state.status,
21177            RunStatus::Landing,
21178            "land could not actually reach the fake pull request, so it must \
21179             have given up rather than left the run silently parked forever"
21180        );
21181        // `land` could not reach the fake pull request, so it gave up into
21182        // `Blocked` - still resumable, so the question must not have been
21183        // swept just because this branch now also calls `settle_questions`.
21184        assert_eq!(runner.state.status, RunStatus::Blocked);
21185        assert!(
21186            store.get(&q.id).unwrap().status.open(),
21187            "Blocked is still alive; settle_questions must have been a no-op here"
21188        );
21189    }
21190
21191    fn state_with_round(round: ReviewRound) -> RunState {
21192        let mut s = RunState::new(
21193            PathBuf::from("/repo"),
21194            "main".to_owned(),
21195            "abc1234".to_owned(),
21196            "add retries".to_owned(),
21197            Config::default(),
21198        );
21199        s.reviews = vec![round];
21200        s
21201    }
21202
21203    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21204        crate::verdict::Finding {
21205            id: id.to_owned(),
21206            severity,
21207            file: None,
21208            line: None,
21209            title: title.to_owned(),
21210            detail: String::new(),
21211        }
21212    }
21213
21214    #[test]
21215    fn pr_body_names_open_findings_and_declined_ones() {
21216        let round = ReviewRound {
21217            round: 2,
21218            head: "deadbee".to_owned(),
21219            verified_head: None,
21220            verified_at: None,
21221            reviews: vec![ReviewRecord {
21222                attempts: 0,
21223                reviewer: 1,
21224                agent: "alpha".to_owned(),
21225                summary: String::new(),
21226                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21227                vote: None,
21228                failed: None,
21229                duration_ms: 0,
21230            }],
21231            e2e: vec![CommandOutcome {
21232                command: "cargo test".to_owned(),
21233                code: Some(0),
21234                output_tail: String::new(),
21235                duration_ms: 0,
21236                resource_blocked: false,
21237            }],
21238            verify_retried: false,
21239            e2e_deferred: false,
21240            e2e_defer_reason: None,
21241            fix: Some(FixRecord {
21242                agent: "alpha".to_owned(),
21243                addressed: Vec::new(),
21244                rejected: vec![crate::verdict::Rejection {
21245                    id: "R1-1-1".to_owned(),
21246                    why: "not reachable from any caller".to_owned(),
21247                }],
21248                notes: String::new(),
21249                committed: true,
21250                failed: None,
21251                duration_ms: 0,
21252                continuation: None,
21253            }),
21254            blocking: 0,
21255            answered: 1,
21256            expected: 1,
21257            clean: false,
21258            progressed: true,
21259            vote_split: false,
21260            reconsideration: Vec::new(),
21261            verdict: None,
21262        };
21263        let state = state_with_round(round);
21264        let body = pr_message(&state, 'A').body;
21265
21266        assert!(body.contains("add retries"), "the task must still be there");
21267        assert!(body.contains("R2-1-1"), "{body}");
21268        assert!(!body.contains("unused import"), "{body}");
21269        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21270        assert!(
21271            !body.contains("not reachable from any caller"),
21272            "the reason it was declined: {body}"
21273        );
21274    }
21275
21276    #[test]
21277    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21278        let round = ReviewRound {
21279            round: 1,
21280            head: "deadbee".to_owned(),
21281            verified_head: None,
21282            verified_at: None,
21283            reviews: vec![ReviewRecord {
21284                attempts: 0,
21285                reviewer: 1,
21286                agent: "alpha".to_owned(),
21287                summary: String::new(),
21288                findings: Vec::new(),
21289                vote: None,
21290                failed: None,
21291                duration_ms: 0,
21292            }],
21293            e2e: Vec::new(),
21294            verify_retried: false,
21295            e2e_deferred: false,
21296            e2e_defer_reason: None,
21297            fix: None,
21298            blocking: 0,
21299            answered: 1,
21300            expected: 1,
21301            clean: true,
21302            progressed: false,
21303            vote_split: false,
21304            reconsideration: Vec::new(),
21305            verdict: None,
21306        };
21307        let state = state_with_round(round);
21308        let body = pr_message(&state, 'A').body;
21309        assert!(!body.contains("Open review findings"), "{body}");
21310        assert!(!body.contains("Declined"), "{body}");
21311    }
21312
21313    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21314        let mut state = RunState::new(
21315            PathBuf::from("/repo"),
21316            "main".to_owned(),
21317            "abc1234".to_owned(),
21318            instruction.to_owned(),
21319            Config::default(),
21320        );
21321        // These fixtures test execution flow, independent of host disk capacity.
21322        state.config.disk.min_free_bytes = 0;
21323        state.candidates.push(Candidate {
21324            index: 0,
21325            label: 'A',
21326            agent: "alpha".to_owned(),
21327            branch: "magi/x/A".to_owned(),
21328            worktree: PathBuf::from("/wt"),
21329            summary: summary.to_owned(),
21330            stat: String::new(),
21331            files: 1,
21332            commits: 1,
21333            empty: false,
21334            failed: None,
21335            verified_noop: None,
21336            folded: false,
21337            duration_ms: 0,
21338        });
21339        state
21340    }
21341
21342    fn review_state(subjects: &[&str]) -> RunState {
21343        let mut state = state_with_summary(
21344            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21345            "",
21346        );
21347        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21348        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21349        state
21350    }
21351
21352    /// A branch rebased onto a main that moved past the recorded
21353    /// `base_commit` is titled from its own first commit, never main's.
21354    #[tokio::test]
21355    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21356        ask_test_home();
21357        let tmp = tempfile::tempdir().unwrap();
21358        let repo = tmp.path().join("repo");
21359        std::fs::create_dir_all(&repo).unwrap();
21360        init_repo(&repo);
21361        let origin = tmp.path().join("origin.git");
21362        let g = |dir: &Path, args: &[&str]| {
21363            let out = std::process::Command::new("git")
21364                .args(args)
21365                .current_dir(dir)
21366                .quiet()
21367                .output()
21368                .expect("spawn git");
21369            assert!(
21370                out.status.success(),
21371                "git {args:?}: {}",
21372                String::from_utf8_lossy(&out.stderr)
21373            );
21374        };
21375        g(
21376            tmp.path(),
21377            &[
21378                "clone",
21379                "--bare",
21380                "-q",
21381                repo.to_str().unwrap(),
21382                origin.to_str().unwrap(),
21383            ],
21384        );
21385        g(
21386            &repo,
21387            &["remote", "add", "origin", origin.to_str().unwrap()],
21388        );
21389        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21390
21391        // Main moves on; the branch is built on top of the new main.
21392        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21393        g(&repo, &["add", "-A"]);
21394        g(
21395            &repo,
21396            &["commit", "-q", "-m", "chore(deps): update a crate"],
21397        );
21398        g(&repo, &["push", "-q", "origin", "main"]);
21399        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21400        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21401        g(&repo, &["add", "-A"]);
21402        g(
21403            &repo,
21404            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21405        );
21406        g(&repo, &["checkout", "-q", "main"]);
21407
21408        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21409        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21410        // Without a readable tracking ref the recorded base's merge base is used.
21411        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21412        assert_eq!(fallback, c1);
21413
21414        let mut state = review_state(&[
21415            "chore(deps): update a crate",
21416            "fix(daemon): apply a chosen action",
21417        ]);
21418        state.repo = repo.clone();
21419        state.base_branch = "main".to_owned();
21420        state.base_commit = c1;
21421        refresh_reviewed_commits(&mut state, "feat/own").await;
21422        assert_eq!(
21423            state.reviewed_commits,
21424            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21425        );
21426        assert_eq!(
21427            review_title(&state).as_deref(),
21428            Some("fix(daemon): apply a chosen action")
21429        );
21430        assert_eq!(
21431            leaked_subjects(&state, "feat/own").await,
21432            Some(vec!["chore(deps): update a crate".to_owned()])
21433        );
21434        // A stale tracking ref is still used when the fetch fails, but the
21435        // leak list is withheld.
21436        state.config.merge.remote = "nowhere".to_owned();
21437        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21438    }
21439
21440    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21441    /// with `origin/main` one commit behind it, and a summarizing agent that
21442    /// prints whatever `reply` holds.
21443    struct SummaryFixture {
21444        _tmp: tempfile::TempDir,
21445        repo: PathBuf,
21446        reply: PathBuf,
21447        runner: Runner,
21448    }
21449
21450    fn summary_fixture() -> SummaryFixture {
21451        ask_test_home();
21452        let tmp = tempfile::tempdir().unwrap();
21453        let repo = tmp.path().join("repo");
21454        std::fs::create_dir_all(&repo).unwrap();
21455        init_repo(&repo);
21456        let origin = tmp.path().join("origin.git");
21457        let g = |args: &[&str]| {
21458            let out = std::process::Command::new("git")
21459                .args(args)
21460                .current_dir(&repo)
21461                .quiet()
21462                .output()
21463                .expect("spawn git");
21464            assert!(
21465                out.status.success(),
21466                "git {args:?}: {}",
21467                String::from_utf8_lossy(&out.stderr)
21468            );
21469        };
21470        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21471        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21472        g(&["push", "-q", "origin", "main"]);
21473        g(&["checkout", "-q", "-b", "feat/own"]);
21474        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21475        g(&["add", "-A"]);
21476        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21477        let reply = tmp.path().join("reply.txt");
21478        std::fs::write(
21479            &reply,
21480            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21481        )
21482        .unwrap();
21483        let mut runner = runner_at(RunStatus::Gating);
21484        let mut state = review_state(&["feat(own): add the own file"]);
21485        state.repo = repo.clone();
21486        state.base_branch = "main".to_owned();
21487        state.candidates[0].branch = "feat/own".to_owned();
21488        state.candidates[0].worktree = repo.clone();
21489        let mut author = spec("alpha");
21490        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21491        author
21492            .env
21493            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21494        state.config.agents = vec![author];
21495        runner.state = state;
21496        SummaryFixture {
21497            _tmp: tmp,
21498            repo,
21499            reply,
21500            runner,
21501        }
21502    }
21503
21504    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21505        let winner = f.runner.state.candidates[0].clone();
21506        f.runner
21507            .guarded_pr_message(&winner, None, true)
21508            .await
21509            .unwrap()
21510            .expect("an English description never parks")
21511    }
21512
21513    fn clean_round_with_minor_findings() -> ReviewRound {
21514        let mut round = passing_round(2, None);
21515        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21516        round.reviews = vec![ReviewRecord {
21517            attempts: 0,
21518            reviewer: 1,
21519            agent: "alpha".into(),
21520            summary: "内部レビュー".into(),
21521            findings: vec![
21522                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21523                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21524            ],
21525            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21526            failed: None,
21527            duration_ms: 0,
21528        }];
21529        round
21530    }
21531
21532    #[tokio::test]
21533    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21534        for review_only in [false, true] {
21535            let mut f = summary_fixture();
21536            if !review_only {
21537                f.runner.state.candidates[0].agent = "alpha".into();
21538            }
21539            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21540            assert!(f.runner.state.reviews[0].clean);
21541            assert!(f.runner.state.reviews[0].fix.is_none());
21542            assert!(f.runner.state.open_findings().is_empty());
21543            assert_eq!(
21544                crate::pr_outcome::required_finding_ids(&f.runner.state),
21545                ["R2-1-1", "R2-1-2"]
21546            );
21547            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21548            let evidence = crate::pr_outcome::summary_evidence(
21549                &f.runner.state,
21550                crate::pr_outcome::FinalHead::At(&head),
21551            );
21552            assert!(
21553                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21554            );
21555            // The ordinary reply omits both IDs: no cached description is adopted.
21556            let omitted = posted(&mut f).await;
21557            assert!(omitted.body.contains("No description"));
21558            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21559            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21560            assert!(f.runner.state.change_summary.is_none());
21561            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21562            let described = posted(&mut f).await;
21563            assert!(described.body.contains("improve documentation for callers"));
21564            assert!(described.body.contains("remaining edge case"));
21565            assert!(described.body.contains("## Open review findings"));
21566            assert!(!described.body.contains("未修正"));
21567        }
21568    }
21569
21570    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21571        let mut f = summary_fixture();
21572        if !review_only {
21573            f.runner.state.candidates[0].agent = "alpha".into();
21574        }
21575        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21576        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21577        fixed
21578            .fix
21579            .as_mut()
21580            .unwrap()
21581            .rejected
21582            .push(crate::verdict::Rejection {
21583                id: "R1-1-1".into(),
21584                why: "呼び出されないコードのため辞退".into(),
21585            });
21586        let mut clean = clean_round_with_minor_findings();
21587        clean.verified_head = Some(head.clone());
21588        f.runner.state.reviews = vec![fixed, clean];
21589        ok_gate(&mut f.runner.state, Some(&head));
21590        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21591        f.runner.state.config.agents.push(scripted("beta", reply));
21592        f.runner.state.config.roles.language_judge =
21593            Some(crate::config::AgentChoice::One("beta".into()));
21594        f
21595    }
21596
21597    #[tokio::test]
21598    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21599        for review_only in [false, true] {
21600            let mut f = rewrite_fixture_with_findings(review_only,
21601                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21602            let m = posted(&mut f).await;
21603            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21604            let expected =
21605                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21606            assert!(m.body.contains(&expected));
21607            assert!(m.body.contains("## Review rounds"));
21608            assert!(
21609                m.body
21610                    .contains("Gate (`verify.gate`): passed on the final commit")
21611            );
21612            assert!(
21613                m.body
21614                    .contains("Tests (`verify.e2e`): passed on the final commit")
21615            );
21616            let c = f.runner.state.change_summary.as_ref().unwrap();
21617            assert!(
21618                m.body
21619                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21620            );
21621            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21622            assert!(crate::pr_outcome::covers_required_findings(
21623                &f.runner.state,
21624                &m.body
21625            ));
21626            assert!(m.body.contains("documentation remains incomplete"));
21627            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21628            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21629            assert!(f.runner.state.github_text.is_none());
21630        }
21631    }
21632
21633    #[tokio::test]
21634    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21635        for review_only in [false, true] {
21636            for reply in [
21637                "TITLE: fix: public description\nA short English description.",
21638                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21639            ] {
21640                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21641                let winner = f.runner.state.candidates[0].clone();
21642                let posted = f
21643                    .runner
21644                    .guarded_pr_message(&winner, None, true)
21645                    .await
21646                    .unwrap();
21647                assert!(posted.is_none());
21648                assert!(f.runner.state.parked);
21649                let withheld = f.runner.state.github_text.as_ref().unwrap();
21650                assert!(
21651                    withheld.rewrite.is_none(),
21652                    "an incomplete rewrite must not be selectable"
21653                );
21654                let q = ask::Questions::open()
21655                    .get(withheld.question.as_ref().unwrap())
21656                    .unwrap();
21657                assert!(
21658                    !q.choices
21659                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21660                );
21661                assert!(
21662                    q.detail.contains("R1-1-1")
21663                        && q.detail.contains("R2-1-1")
21664                        && q.detail.contains("R2-1-2")
21665                );
21666            }
21667        }
21668    }
21669
21670    #[tokio::test]
21671    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21672        for review_only in [false, true] {
21673            let mut f = summary_fixture();
21674            if !review_only {
21675                f.runner.state.candidates[0].agent = "alpha".into();
21676            }
21677            f.runner.state.instruction = "内部タスクの説明".into();
21678            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21679            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21680            f.runner.state.reviews = vec![
21681                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21682                round_with_notes(2, Some("二回目の内部修正報告です。")),
21683                passing_round(3, Some(&head)),
21684            ];
21685            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21686            ok_gate(&mut f.runner.state, Some(&head));
21687            let captured = f.repo.join("public-prompt.txt");
21688            let author = &mut f.runner.state.config.agents[0];
21689            author.command = vec![
21690                "sh".into(),
21691                "-c".into(),
21692                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21693            ];
21694            author.env.insert(
21695                "PROMPT_FILE".into(),
21696                captured.to_string_lossy().into_owned(),
21697            );
21698            let m = posted(&mut f).await;
21699            assert!(m.body.contains("Adds `own.txt`"));
21700            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21701            assert!(m.body.contains("R1-1-1"));
21702            assert!(
21703                m.body
21704                    .contains("Gate (`verify.gate`): passed on the final commit")
21705            );
21706            assert!(
21707                m.body
21708                    .contains("Tests (`verify.e2e`): passed on the final commit")
21709            );
21710            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21711            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21712            let prompt = std::fs::read_to_string(captured).unwrap();
21713            assert!(prompt.contains(&format!(
21714                "git diff {} {head}",
21715                f.runner.state.change_summary.as_ref().unwrap().base
21716            )));
21717            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21718            assert_eq!(
21719                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21720                "二回目の内部修正報告です。"
21721            );
21722        }
21723    }
21724
21725    #[tokio::test]
21726    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21727        for review_only in [false, true] {
21728            let mut f = summary_fixture();
21729            if !review_only {
21730                f.runner.state.candidates[0].agent = "alpha".into();
21731            }
21732            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21733            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21734            std::fs::write(&f.reply, "no usable description").unwrap();
21735            let failed = posted(&mut f).await;
21736            assert!(
21737                failed
21738                    .body
21739                    .contains("No description of what this change does could be generated")
21740            );
21741            assert!(failed.body.is_ascii());
21742            assert!(f.runner.state.github_text.is_none());
21743            assert!(f.runner.state.change_summary.is_none());
21744            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21745            let evidence = crate::pr_outcome::summary_evidence(
21746                &f.runner.state,
21747                crate::pr_outcome::FinalHead::At(&head),
21748            );
21749            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21750            assert!(
21751                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21752                    .exists()
21753            );
21754            f.runner.state =
21755                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21756            std::fs::write(
21757                &f.reply,
21758                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21759            )
21760            .unwrap();
21761            let retried = posted(&mut f).await;
21762            assert!(retried.body.contains("Adds the committed own file."));
21763            assert_eq!(
21764                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21765                "日本語の修正報告"
21766            );
21767        }
21768    }
21769
21770    #[tokio::test]
21771    async fn same_head_verification_and_findings_invalidate_public_prose() {
21772        let mut f = summary_fixture();
21773        posted(&mut f).await;
21774        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21775        let original = f
21776            .runner
21777            .state
21778            .change_summary
21779            .as_ref()
21780            .unwrap()
21781            .evidence
21782            .clone();
21783        ok_gate(&mut f.runner.state, Some(&head));
21784        std::fs::write(
21785            &f.reply,
21786            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21787        )
21788        .unwrap();
21789        let verified = posted(&mut f).await;
21790        assert!(verified.body.contains("now verified"));
21791        assert_ne!(
21792            f.runner.state.change_summary.as_ref().unwrap().evidence,
21793            original
21794        );
21795        let mut round = round_with_notes(1, Some("内部報告"));
21796        round.reviews.push(ReviewRecord {
21797            attempts: 0,
21798            reviewer: 1,
21799            agent: "alpha".into(),
21800            summary: "内部レビュー".into(),
21801            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21802            vote: None,
21803            failed: None,
21804            duration_ms: 0,
21805        });
21806        round
21807            .fix
21808            .as_mut()
21809            .unwrap()
21810            .rejected
21811            .push(crate::verdict::Rejection {
21812                id: "R1-1-1".into(),
21813                why: "呼び出し元から到達できない".into(),
21814            });
21815        f.runner.state.reviews = vec![round];
21816        // A description omitting the required ID is unusable, even with a title.
21817        let omitted = posted(&mut f).await;
21818        assert!(omitted.body.contains("No description"));
21819        assert!(omitted.body.contains("R1-1-1"));
21820        assert!(!omitted.body.contains("到達"));
21821        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21822        let described = posted(&mut f).await;
21823        assert!(described.body.contains("unreachable from callers"));
21824        assert!(described.body.contains("R1-1-1"));
21825        assert!(!described.body.contains("到達"));
21826        let evidence = f
21827            .runner
21828            .state
21829            .change_summary
21830            .as_ref()
21831            .unwrap()
21832            .evidence
21833            .clone();
21834        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21835        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21836        assert!(posted(&mut f).await.body.contains("rationale updated"));
21837        assert_ne!(
21838            f.runner.state.change_summary.as_ref().unwrap().evidence,
21839            evidence
21840        );
21841        // Old serialized summaries have no evidence fingerprint and must regenerate.
21842        f.runner
21843            .state
21844            .change_summary
21845            .as_mut()
21846            .unwrap()
21847            .evidence
21848            .clear();
21849        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21850        assert!(posted(&mut f).await.body.contains("current specification"));
21851    }
21852
21853    #[tokio::test]
21854    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21855        let mut f = summary_fixture();
21856        let m = posted(&mut f).await;
21857        assert_eq!(m.title, "feat(own): add the own file");
21858        assert!(
21859            m.body
21860                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21861            "{}",
21862            m.body
21863        );
21864        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21865        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21866        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21867        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21868        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21869        let kept = f.runner.state.change_summary.as_ref().unwrap();
21870        assert_eq!(
21871            (kept.head.as_str(), kept.base.as_str()),
21872            (head.as_str(), base.as_str())
21873        );
21874        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21875    }
21876
21877    #[tokio::test]
21878    async fn review_only_summary_receives_the_configured_github_language() {
21879        let mut f = summary_fixture();
21880        f.runner.state.config.graph.github_language = "ja".into();
21881        let captured = f.repo.join("summary-prompt.txt");
21882        let author = &mut f.runner.state.config.agents[0];
21883        author.command = vec![
21884            "sh".into(),
21885            "-c".into(),
21886            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21887        ];
21888        author.env.insert(
21889            "PROMPT_FILE".into(),
21890            captured.to_string_lossy().into_owned(),
21891        );
21892        std::fs::write(
21893            &f.reply,
21894            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21895        )
21896        .unwrap();
21897        let m = posted(&mut f).await;
21898        let prompt = std::fs::read_to_string(captured).unwrap();
21899        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21900        assert!(!prompt.contains("in English"));
21901        assert_eq!(m.title, "feat(own): ファイルを追加");
21902        assert!(m.body.contains("own.txt を追加します。"));
21903    }
21904
21905    #[tokio::test]
21906    async fn review_fixes_come_after_the_whole_change_summary() {
21907        let mut f = summary_fixture();
21908        f.runner.state.reviews = vec![
21909            round_with_notes(1, Some("handled the PATH case")),
21910            round_with_notes(2, None),
21911        ];
21912        let m = posted(&mut f).await;
21913        let summary = m.body.find("## Summary").expect("summary");
21914        let fixes = m.body.find("## Review rounds").expect("review rounds");
21915        assert!(summary < fixes, "{}", m.body);
21916        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21917        assert!(m.body.contains("## Final verification"), "{}", m.body);
21918    }
21919
21920    #[tokio::test]
21921    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21922        let mut f = summary_fixture();
21923        let first = posted(&mut f).await;
21924        let first_head = f.runner.state.change_summary.clone().unwrap().head;
21925        // The same head reuses the record: a different reply is not read.
21926        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21927        let again = posted(&mut f).await;
21928        assert_eq!((again.title, again.body), (first.title, first.body));
21929
21930        let g = |args: &[&str]| {
21931            let out = std::process::Command::new("git")
21932                .args(args)
21933                .current_dir(&f.repo)
21934                .quiet()
21935                .output()
21936                .expect("spawn git");
21937            assert!(out.status.success(), "git {args:?}");
21938        };
21939        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21940        g(&["add", "-A"]);
21941        g(&["commit", "-q", "-m", "fix(own): handle more"]);
21942        let second = posted(&mut f).await;
21943        let kept = f.runner.state.change_summary.clone().unwrap();
21944        assert_ne!(kept.head, first_head);
21945        assert!(second.body.contains("Not read yet."), "{}", second.body);
21946        assert!(!second.body.contains("so the first summary is read"));
21947        assert_eq!(second.title, "feat(own): other");
21948    }
21949
21950    #[tokio::test]
21951    async fn an_unavailable_summary_is_stated_and_never_invented() {
21952        let mut f = summary_fixture();
21953        std::fs::write(&f.reply, "no title line here").unwrap();
21954        let m = posted(&mut f).await;
21955        assert!(
21956            m.body
21957                .contains("No description of what this change does could be generated"),
21958            "{}",
21959            m.body
21960        );
21961        assert!(m.body.contains("internal reports remain in local records"));
21962        assert!(f.runner.state.change_summary.is_none());
21963        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21964        // A preview asks nobody and claims nothing.
21965        let winner = f.runner.state.candidates[0].clone();
21966        let preview = f
21967            .runner
21968            .guarded_pr_message(&winner, None, false)
21969            .await
21970            .unwrap()
21971            .unwrap();
21972        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21973    }
21974
21975    #[test]
21976    fn pr_message_review_single_commit_uses_its_subject() {
21977        let state = review_state(&["feat(nats): per-role user"]);
21978        let m = pr_message(&state, 'A');
21979        assert_eq!(m.title, "feat(nats): per-role user");
21980        assert!(!m.body.contains("Review the work already"), "{}", m.body);
21981        assert!(m.body.contains("## Commits under review"), "{}", m.body);
21982    }
21983
21984    #[test]
21985    fn pr_message_review_multi_commit_takes_the_oldest() {
21986        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21987        let m = pr_message(&state, 'A');
21988        assert_eq!(m.title, "feat: the change");
21989        for s in ["feat: the change", "fix: typo", "fix: again"] {
21990            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21991        }
21992    }
21993
21994    #[test]
21995    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21996        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21997            let mut state = review_state(&[first, "fix: later fixup"]);
21998            state.candidates[0].branch = "機能/ログイン".to_owned();
21999            let m = pr_message(&state, 'A');
22000            assert!(
22001                m.title.starts_with("chore: land candidate A of run"),
22002                "{}",
22003                m.title
22004            );
22005        }
22006    }
22007
22008    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22009        BranchFacts {
22010            commits: commits
22011                .iter()
22012                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22013                .collect(),
22014            stat: stat.to_owned(),
22015        }
22016    }
22017
22018    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22019        let mut r = review_round(true, 0, 1, 1, true, true);
22020        r.round = round;
22021        r.fix = notes.map(|n| FixRecord {
22022            agent: "fixer".to_owned(),
22023            addressed: Vec::new(),
22024            rejected: Vec::new(),
22025            notes: n.to_owned(),
22026            committed: true,
22027            failed: None,
22028            duration_ms: 0,
22029            continuation: None,
22030        });
22031        r
22032    }
22033
22034    #[test]
22035    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22036        let state = review_state(&["ignored"]);
22037        let f = facts(
22038            &[
22039                (
22040                    "fix(login): resolve PATH on macOS",
22041                    "Login shells skip rc files.",
22042                ),
22043                ("fix: address review", ""),
22044            ],
22045            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22046        );
22047        let m = pr_message_with(&state, 'A', Some(&f));
22048        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22049        assert!(
22050            m.body
22051                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22052            "{}",
22053            m.body
22054        );
22055        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22056        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22057        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22058        for banned in [
22059            "Review the work already",
22060            "no task statement",
22061            "Original task",
22062        ] {
22063            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22064        }
22065        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22066    }
22067
22068    #[test]
22069    fn pr_message_review_truncates_a_huge_first_commit_body() {
22070        let state = review_state(&["ignored"]);
22071        let f = facts(
22072            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22073            "s",
22074        );
22075        let m = pr_message_with(&state, 'A', Some(&f));
22076        assert!(m.body.len() < 30_000, "{}", m.body.len());
22077        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22078        assert!(
22079            m.body.contains("- ... 1 more commit(s)"),
22080            "{}",
22081            m.body.len()
22082        );
22083        assert!(m.body.ends_with("magi:candidate-a\n"));
22084    }
22085
22086    #[test]
22087    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22088        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22089        assert_eq!(m.title, "feat: x");
22090        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22091        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22092        assert!(!m.body.contains("no task statement"), "{}", m.body);
22093    }
22094
22095    #[test]
22096    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22097        let mut state = review_state(&["日本語の件名"]);
22098        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22099        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22100        state.candidates[0].branch = "機能/ログイン".to_owned();
22101        assert!(
22102            pr_message(&state, 'A')
22103                .title
22104                .starts_with("chore: land candidate A")
22105        );
22106    }
22107
22108    #[test]
22109    fn pr_message_review_fixes_survive_a_clean_final_round() {
22110        let mut state = review_state(&["feat: x"]);
22111        state.reviews = vec![
22112            round_with_notes(1, Some("handled the PATH case")),
22113            round_with_notes(2, None),
22114        ];
22115        let body = pr_message(&state, 'A').body;
22116        assert!(
22117            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22118            "{body}"
22119        );
22120        assert!(body.contains("### Round 1"), "{body}");
22121
22122        state.reviews = vec![
22123            round_with_notes(1, Some("first fix")),
22124            round_with_notes(2, Some("")),
22125            round_with_notes(3, Some("second fix")),
22126            round_with_notes(4, None),
22127        ];
22128        let body = pr_message(&state, 'A').body;
22129        assert!(
22130            body.contains("### Round 1") && !body.contains("first fix"),
22131            "{body}"
22132        );
22133        assert!(
22134            body.contains("### Round 3") && !body.contains("second fix"),
22135            "{body}"
22136        );
22137        assert!(body.contains("### Round 2"), "{body}");
22138    }
22139
22140    const FINAL: &str = "abcdef1234567890";
22141
22142    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22143        let mut r = round_with_notes(round, None);
22144        r.verified_head = verified.map(str::to_owned);
22145        r
22146    }
22147
22148    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22149        state.gate = vec![CommandOutcome {
22150            command: "gate".to_owned(),
22151            code: Some(0),
22152            output_tail: String::new(),
22153            duration_ms: 0,
22154            resource_blocked: false,
22155        }];
22156        state.gate_ran = true;
22157        state.gate_head = head.map(str::to_owned);
22158    }
22159
22160    fn body_at(state: &RunState) -> String {
22161        pr_message_raw(
22162            state,
22163            'A',
22164            None,
22165            crate::pr_outcome::FinalHead::At(FINAL),
22166            ChangeNote::NotAsked,
22167        )
22168        .body
22169    }
22170
22171    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22172
22173    #[test]
22174    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22175        let mut state = review_state(&["feat: x"]);
22176        state.reviews = vec![
22177            round_with_notes(1, Some(SANDBOXED)),
22178            passing_round(2, Some(FINAL)),
22179        ];
22180        ok_gate(&mut state, Some(FINAL));
22181        let body = body_at(&state);
22182        assert!(!body.contains("Handled the PATH case."), "{body}");
22183        assert!(!body.contains("could not commit"), "{body}");
22184        assert!(!body.contains("could not run"), "{body}");
22185        assert!(
22186            body.contains("The fix is committed on the branch"),
22187            "{body}"
22188        );
22189        assert!(!body.contains("sandbox"), "{body}");
22190        assert!(
22191            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22192            "{body}"
22193        );
22194        assert!(
22195            body.contains("Gate (`verify.gate`): passed on the final commit"),
22196            "{body}"
22197        );
22198        // The raw report is untouched in the record.
22199        assert!(
22200            state.reviews[0]
22201                .fix
22202                .as_ref()
22203                .unwrap()
22204                .notes
22205                .contains("could not commit")
22206        );
22207    }
22208
22209    #[test]
22210    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22211        let mut state = review_state(&["feat: x"]);
22212        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22213        r1.fix.as_mut().unwrap().committed = false;
22214        let mut r2 = passing_round(2, Some(FINAL));
22215        r2.e2e[0].code = Some(1);
22216        state.reviews = vec![r1, r2];
22217        let body = body_at(&state);
22218        assert!(!body.contains("could not commit"), "{body}");
22219        assert!(!body.contains("could not run"), "{body}");
22220        assert!(body.contains("This round produced no commit"), "{body}");
22221        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22222        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22223        assert!(!body.contains("passed"), "{body}");
22224    }
22225
22226    #[test]
22227    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22228        let mut state = review_state(&["feat: x"]);
22229        state.reviews = vec![
22230            round_with_notes(1, Some(SANDBOXED)),
22231            passing_round(2, Some("0123456789abcdef")),
22232        ];
22233        ok_gate(&mut state, Some("0123456789abcdef"));
22234        let body = body_at(&state);
22235        assert!(!body.contains("HTTP tests could not run"), "{body}");
22236        // The commit claim is still settled by the structured record.
22237        assert!(!body.contains("could not commit"), "{body}");
22238        assert!(
22239            body.contains("passed on an earlier commit (`0123456`)"),
22240            "{body}"
22241        );
22242        assert!(!body.contains("passed on the final commit"), "{body}");
22243
22244        // A gate recorded before `gate_head` existed is "not recorded".
22245        ok_gate(&mut state, None);
22246        let body = body_at(&state);
22247        assert!(
22248            body.contains("passed on a commit that was not recorded"),
22249            "{body}"
22250        );
22251    }
22252
22253    #[test]
22254    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22255        let state = state_with_summary(
22256            "add retries to the client",
22257            "TITLE: feat: retries\n\nDid it.",
22258        );
22259        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22260        assert_eq!(m.title, "feat: retries");
22261        assert!(
22262            m.body.contains("<summary>Original task</summary>"),
22263            "{}",
22264            m.body
22265        );
22266        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22267        assert!(
22268            m.body
22269                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22270            "{}",
22271            m.body
22272        );
22273    }
22274
22275    #[test]
22276    fn pr_message_review_bounds_a_long_english_subject() {
22277        let long = format!("feat: {}", "word ".repeat(100));
22278        let m = pr_message(&review_state(&[&long]), 'A');
22279        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22280        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22281    }
22282
22283    #[test]
22284    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22285        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22286        let m = pr_message(&state, 'A');
22287        assert_eq!(m.title, "add retries");
22288        assert!(m.body.contains("<summary>Original task</summary>"));
22289        assert!(!m.body.contains("Commits under review"));
22290        assert_eq!(landing_subject_source(&state), state.instruction);
22291    }
22292
22293    #[test]
22294    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22295        let state = review_state(&["feat: the change", "fix: typo"]);
22296        let source = landing_subject_source(&state);
22297        assert_eq!(land::merge_subject("", &source), "feat: the change");
22298        assert_eq!(
22299            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22300            "feat: the change"
22301        );
22302        // An operator's rename still wins.
22303        assert_eq!(
22304            land::merge_subject("feat: renamed by hand", &source),
22305            "feat: renamed by hand"
22306        );
22307        let blank = review_state(&["日本語"]);
22308        assert!(
22309            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22310        );
22311    }
22312
22313    #[test]
22314    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22315        let state = review_state(&["feat: the change"]);
22316        let source = landing_subject_source(&state);
22317        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22318        assert_eq!(
22319            land::merge_subject(landing_title(&state, old), &source),
22320            "feat: the change"
22321        );
22322        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22323        let task = state_with_summary("add retries", "");
22324        assert_eq!(landing_title(&task, old), old);
22325    }
22326
22327    #[test]
22328    fn pr_message_describes_the_change_not_the_task() {
22329        let state = state_with_summary(
22330            "今回やってほしいこと: results projector を直す",
22331            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22332        );
22333        let m = pr_message(&state, 'A');
22334        assert_eq!(m.title, "fix(web): batch the runs list reads");
22335        assert!(
22336            m.body.starts_with("## Summary\n\n- reads run.json once"),
22337            "{}",
22338            m.body
22339        );
22340        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22341        let task_at = m.body.find("今回やってほしいこと").unwrap();
22342        let details_at = m.body.find("<details>").unwrap();
22343        assert!(
22344            details_at < task_at,
22345            "the task lives inside <details>: {}",
22346            m.body
22347        );
22348        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22349        assert!(m.body.contains("magi:candidate-a"));
22350    }
22351
22352    #[test]
22353    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22354        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22355        let m = pr_message(&state, 'A');
22356        assert_eq!(m.title, "add retries");
22357        assert!(
22358            m.body.contains("## Summary\n\n- did some things"),
22359            "{}",
22360            m.body
22361        );
22362
22363        let none = RunState::new(
22364            PathBuf::from("/repo"),
22365            "main".to_owned(),
22366            "abc1234".to_owned(),
22367            "add retries".to_owned(),
22368            Config::default(),
22369        );
22370        let m = pr_message(&none, 'A');
22371        assert_eq!(m.title, "add retries");
22372        assert!(!m.body.contains("## Summary"), "{}", m.body);
22373    }
22374
22375    #[test]
22376    fn pr_message_refuses_the_candidate_commit_subject() {
22377        for bad in [
22378            "TITLE: magi: candidate A (uncommitted work)",
22379            "TITLE: chore: stuff (uncommitted work)",
22380            "TITLE:   ",
22381        ] {
22382            let state = state_with_summary("add retries", bad);
22383            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22384        }
22385    }
22386
22387    #[test]
22388    fn pr_message_bounds_a_very_long_task_and_title() {
22389        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22390        let state = state_with_summary(&long, "- nothing");
22391        let m = pr_message(&state, 'A');
22392        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22393        assert!(!m.title.contains('\n'));
22394
22395        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22396        let m = pr_message(&state, 'A');
22397        assert!(m.title.starts_with("feat: "));
22398        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22399        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22400    }
22401
22402    fn long_title_of(instruction: &str) -> String {
22403        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22404    }
22405
22406    #[test]
22407    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22408        let first = "Make the landing path keep a readable title for long tasks";
22409        let line = format!(
22410            "{first}. {}",
22411            "And then keep going with more words ".repeat(20)
22412        );
22413        let t = long_title_of(&line);
22414        assert_eq!(t, first);
22415        assert!(!t.starts_with("chore: land"));
22416    }
22417
22418    #[test]
22419    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22420        let line = "word ".repeat(200);
22421        let t = long_title_of(&line);
22422        assert!(t.ends_with("word..."), "{t}");
22423        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22424    }
22425
22426    #[test]
22427    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22428        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22429            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22430        }
22431    }
22432
22433    #[test]
22434    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22435        let mut state = state_with_summary("再試行を追加する", "- nothing");
22436        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22437        state.config.graph.github_language = "ja".into();
22438        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22439    }
22440
22441    #[test]
22442    fn pr_message_title_limit_is_exact() {
22443        let at = "a".repeat(PR_TITLE_MAX);
22444        assert_eq!(long_title_of(&at), at);
22445        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22446        assert!(over.ends_with("..."), "{over}");
22447        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22448    }
22449
22450    #[test]
22451    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22452        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22453        let t = long_title_of(&line);
22454        assert!(t.ends_with("..."), "{t}");
22455        assert!(t.is_ascii(), "{t}");
22456    }
22457
22458    #[test]
22459    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22460        let line = format!(
22461            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22462            "plus more filler words ".repeat(20)
22463        );
22464        let t = long_title_of(&line);
22465        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22466    }
22467
22468    #[test]
22469    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22470        let line = format!("\n\n# {}", "title words ".repeat(40));
22471        let state = state_with_summary(&line, "- nothing");
22472        let m = pr_message(&state, 'A');
22473        assert!(m.title.starts_with("title words"), "{}", m.title);
22474        assert_eq!(
22475            land::merge_subject(&m.title, &landing_subject_source(&state)),
22476            m.title
22477        );
22478        // An operator's rename wins untouched.
22479        assert_eq!(
22480            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22481            "feat: renamed by hand"
22482        );
22483    }
22484
22485    #[tokio::test]
22486    async fn github_text_rewrite_is_bounded_and_falls_back() {
22487        crate::run::pin_test_home();
22488        for (reply, accepted) in [
22489            (
22490                "TITLE: fix: retries\nAdd retries for failed requests.",
22491                true,
22492            ),
22493            (
22494                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22495                false,
22496            ),
22497            (
22498                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22499                true,
22500            ),
22501            (
22502                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22503                true,
22504            ),
22505            (
22506                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22507                true,
22508            ),
22509            (
22510                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22511                false,
22512            ),
22513            ("TITLE: fix: retries\n", false),
22514        ] {
22515            let dir = tempfile::tempdir().unwrap();
22516            let mut runner = runner_at(RunStatus::Gating);
22517            runner.state = state_with_summary(
22518                "add retries",
22519                "TITLE: fix: retries\n日本語の説明を書きます。",
22520            );
22521            runner.state.repo = dir.path().to_owned();
22522            runner.state.candidates[0].worktree = dir.path().to_owned();
22523            let mut author = spec("alpha");
22524            author.command = vec![
22525                "sh".into(),
22526                "-c".into(),
22527                "printf '%s' \"$REWRITE_REPLY\"".into(),
22528            ];
22529            author.env.insert("REWRITE_REPLY".into(), reply.into());
22530            runner.state.config.agents = vec![author];
22531            seed_public_summary(&mut runner);
22532            let winner = runner.state.candidates[0].clone();
22533            let first = runner
22534                .guarded_pr_message(&winner, None, true)
22535                .await
22536                .unwrap();
22537            let message = if accepted {
22538                first.expect("a passing rewrite never asks")
22539            } else {
22540                // Withheld: the run parks on a question and posts nothing yet.
22541                assert!(first.is_none());
22542                assert!(runner.state.parked);
22543                let store = ask::Questions::open();
22544                let asked: Vec<_> = store
22545                    .list()
22546                    .into_iter()
22547                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22548                    .collect();
22549                assert_eq!(asked.len(), 1);
22550                assert!(!asked[0].detail.contains("secret"));
22551                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22552                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22553                assert_eq!(
22554                    asked[0]
22555                        .choices
22556                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22557                    rewrite.is_some()
22558                );
22559                // Silence: the question lapses and the neutral text is used,
22560                // without a second rewrite call or a second question.
22561                store
22562                    .update(&asked[0].id, |q| {
22563                        q.abandon("test");
22564                        Ok(())
22565                    })
22566                    .unwrap();
22567                let again = runner
22568                    .guarded_pr_message(&winner, None, true)
22569                    .await
22570                    .unwrap()
22571                    .expect("an abandoned question falls back");
22572                let n = store
22573                    .list()
22574                    .into_iter()
22575                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22576                    .count();
22577                assert_eq!(n, 1, "asked once per rejected text");
22578                again
22579            };
22580            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22581            assert!(!message.body.contains("secret"));
22582            assert!(!message.body.contains("prefix"));
22583            assert!(!message.title.contains("secret"));
22584            assert_eq!(
22585                message.body.contains("Add retries for failed requests."),
22586                accepted
22587            );
22588            assert_eq!(
22589                runner.state.seats["github-text-rewrite"].turns, 1,
22590                "only one rewrite invocation"
22591            );
22592            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22593            if accepted {
22594                assert!(runner.state.github_text.is_none());
22595                assert!(!runner.state.parked);
22596                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22597                assert!(message.body.contains("## Final verification"));
22598                assert!(message.body.contains("Evidence: final diff"));
22599                assert!(
22600                    message
22601                        .body
22602                        .contains(&format!("magi:run/{}", runner.state.id))
22603                );
22604            }
22605            if !accepted {
22606                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22607                assert!(
22608                    message
22609                        .body
22610                        .contains(&format!("magi:run/{}", runner.state.id))
22611                );
22612            }
22613        }
22614    }
22615
22616    fn scripted(id: &str, reply: &str) -> AgentSpec {
22617        let mut a = spec(id);
22618        a.command = vec![
22619            "sh".into(),
22620            "-c".into(),
22621            "printf '%s' \"$REWRITE_REPLY\"".into(),
22622        ];
22623        a.env.insert("REWRITE_REPLY".into(), reply.into());
22624        a
22625    }
22626
22627    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22628    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22629
22630    /// Gate tests inject generated public output, not internal reports.
22631    fn seed_public_summary(runner: &mut Runner) {
22632        let repo = &runner.state.repo;
22633        let g = |args: &[&str]| {
22634            let out = std::process::Command::new("git")
22635                .args(args)
22636                .current_dir(repo)
22637                .quiet()
22638                .output()
22639                .unwrap();
22640            assert!(out.status.success(), "git {args:?}");
22641            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22642        };
22643        g(&["init", "-q", "-b", "main"]);
22644        g(&[
22645            "-c",
22646            "user.name=Test",
22647            "-c",
22648            "user.email=test@example.invalid",
22649            "commit",
22650            "-q",
22651            "--allow-empty",
22652            "-m",
22653            "feat: fixture",
22654        ]);
22655        let head = g(&["rev-parse", "HEAD"]);
22656        g(&["update-ref", "refs/remotes/origin/main", &head]);
22657        let evidence = crate::pr_outcome::summary_evidence(
22658            &runner.state,
22659            crate::pr_outcome::FinalHead::At(&head),
22660        );
22661        let generated = &runner.state.candidates[0].summary;
22662        runner.state.change_summary = Some(crate::run::ChangeSummary {
22663            base: head.clone(),
22664            head,
22665            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22666            title: summary_title(generated).unwrap(),
22667            body: summary_without_title(generated),
22668        });
22669    }
22670
22671    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22672        let mut runner = runner_at(RunStatus::Gating);
22673        runner.state = state_with_summary("add retries", JA_SUMMARY);
22674        if review_only {
22675            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22676        }
22677        runner.state.repo = dir.to_owned();
22678        runner.state.candidates[0].worktree = dir.to_owned();
22679        runner.state.config.agents = agents;
22680        seed_public_summary(&mut runner);
22681        runner
22682    }
22683
22684    fn gate_questions(runner: &Runner) -> usize {
22685        ask::Questions::open()
22686            .list()
22687            .into_iter()
22688            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22689            .count()
22690    }
22691
22692    #[tokio::test]
22693    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22694        crate::run::pin_test_home();
22695        let dir = tempfile::tempdir().unwrap();
22696        let mut runner = rewrite_runner(
22697            dir.path(),
22698            true,
22699            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22700        );
22701        runner.state.config.roles.language_judge =
22702            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22703        let winner = runner.state.candidates[0].clone();
22704        let msg = runner
22705            .guarded_pr_message(&winner, None, true)
22706            .await
22707            .unwrap()
22708            .expect("an English rewrite posts without asking");
22709        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22710        assert!(msg.body.contains("<details>kept</details>"));
22711        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22712        assert_eq!(gate_questions(&runner), 0);
22713        assert!(runner.state.github_text.is_none());
22714        assert!(!runner.state.seats.contains_key("impl-A"));
22715        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22716    }
22717
22718    #[tokio::test]
22719    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22720        crate::run::pin_test_home();
22721        let dir = tempfile::tempdir().unwrap();
22722        let mut runner = rewrite_runner(
22723            dir.path(),
22724            false,
22725            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22726        );
22727        runner.state.config.roles.language_judge =
22728            Some(crate::config::AgentChoice::One("beta".into()));
22729        runner.state.seats.insert(
22730            "impl-A".into(),
22731            SeatState::new("impl-A", "alpha", runner.state.seed),
22732        );
22733        let winner = runner.state.candidates[0].clone();
22734        let msg = runner
22735            .guarded_pr_message(&winner, None, true)
22736            .await
22737            .unwrap()
22738            .expect("rewritten");
22739        assert!(msg.body.contains("<details>kept</details>"));
22740        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22741        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22742        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22743    }
22744
22745    #[tokio::test]
22746    async fn rewrite_reuses_a_resumable_implementer_session() {
22747        crate::run::pin_test_home();
22748        let dir = tempfile::tempdir().unwrap();
22749        let mut runner = rewrite_runner(
22750            dir.path(),
22751            false,
22752            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22753        );
22754        runner.state.config.roles.language_judge =
22755            Some(crate::config::AgentChoice::One("beta".into()));
22756        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22757        seat.turns = 1;
22758        runner.state.seats.insert("impl-A".into(), seat);
22759        let winner = runner.state.candidates[0].clone();
22760        runner
22761            .guarded_pr_message(&winner, None, true)
22762            .await
22763            .unwrap()
22764            .expect("rewritten");
22765        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22766        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22767    }
22768
22769    #[tokio::test]
22770    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22771        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22772        // session, was out of quota, and nothing else was asked.
22773        crate::run::pin_test_home();
22774        let dir = tempfile::tempdir().unwrap();
22775        let mut exhausted = spec("alpha");
22776        exhausted.command = vec!["false".into()];
22777        let mut runner = rewrite_runner(
22778            dir.path(),
22779            false,
22780            vec![exhausted, scripted("beta", EN_REWRITE)],
22781        );
22782        // The implementer's own agent is in the role too: it is not asked twice.
22783        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22784            "alpha".into(),
22785            "beta".into(),
22786        ]));
22787        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22788        seat.turns = 1;
22789        runner.state.seats.insert("impl-A".into(), seat);
22790        let winner = runner.state.candidates[0].clone();
22791        let msg = runner
22792            .guarded_pr_message(&winner, None, true)
22793            .await
22794            .unwrap()
22795            .expect("the role rewrites after the implementer fails");
22796        assert!(msg.body.contains("`src/net.rs`"));
22797        assert_eq!(
22798            runner.state.seats["impl-A"].turns, 2,
22799            "the implementer was tried once"
22800        );
22801        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22802        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22803    }
22804
22805    #[tokio::test]
22806    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22807        crate::run::pin_test_home();
22808        let dir = tempfile::tempdir().unwrap();
22809        let mut broken = spec("bad");
22810        broken.command = vec!["false".into()];
22811        let mut runner = rewrite_runner(
22812            dir.path(),
22813            true,
22814            vec![
22815                broken,
22816                scripted("noformat", "Add retries."),
22817                scripted("good", EN_REWRITE),
22818            ],
22819        );
22820        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22821            "bad".into(),
22822            "noformat".into(),
22823            "good".into(),
22824        ]));
22825        let winner = runner.state.candidates[0].clone();
22826        let msg = runner
22827            .guarded_pr_message(&winner, None, true)
22828            .await
22829            .unwrap()
22830            .expect("the third agent rewrites");
22831        assert!(msg.body.contains("`src/net.rs`"));
22832        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22833        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22834    }
22835
22836    #[tokio::test]
22837    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22838        crate::run::pin_test_home();
22839        // (agents, role): still Japanese; no TITLE; nothing resolvable.
22840        for (agents, role, saved) in [
22841            (
22842                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22843                Some("beta"),
22844                true,
22845            ),
22846            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22847            (Vec::new(), None, false),
22848        ] {
22849            let dir = tempfile::tempdir().unwrap();
22850            let mut runner = rewrite_runner(dir.path(), true, agents);
22851            runner.state.config.roles.language_judge =
22852                role.map(|r| crate::config::AgentChoice::One(r.into()));
22853            let winner = runner.state.candidates[0].clone();
22854            let first = runner
22855                .guarded_pr_message(&winner, None, true)
22856                .await
22857                .unwrap();
22858            assert!(first.is_none(), "withheld text parks the run");
22859            assert!(runner.state.parked);
22860            assert_eq!(gate_questions(&runner), 1);
22861            let g = runner.state.github_text.as_ref().unwrap();
22862            assert_eq!(g.rewrite.is_some(), saved);
22863            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22864        }
22865    }
22866
22867    #[tokio::test]
22868    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22869        crate::run::pin_test_home();
22870        let dir = tempfile::tempdir().unwrap();
22871        let mut runner = runner_at(RunStatus::Gating);
22872        runner.state =
22873            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22874        runner.state.repo = dir.path().to_owned();
22875        runner.state.candidates[0].worktree = dir.path().to_owned();
22876        let mut author = spec("alpha");
22877        author.command = vec![
22878            "sh".into(),
22879            "-c".into(),
22880            "printf '%s' \"$REWRITE_REPLY\"".into(),
22881        ];
22882        author.env.insert(
22883            "REWRITE_REPLY".into(),
22884            format!(
22885                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22886                "English sentence. ".repeat(600)
22887            ),
22888        );
22889        runner.state.config.agents = vec![author];
22890        seed_public_summary(&mut runner);
22891        let winner = runner.state.candidates[0].clone();
22892        assert!(
22893            runner
22894                .guarded_pr_message(&winner, None, true)
22895                .await
22896                .unwrap()
22897                .is_none()
22898        );
22899        let record = runner.state.github_text.clone().unwrap();
22900        let rewrite = record.rewrite.clone().unwrap();
22901        assert_eq!(rewrite.failed, ["body-language"]);
22902        assert!(
22903            rewrite
22904                .body
22905                .ends_with(&format!("magi:run/{}", runner.state.id))
22906        );
22907        let store = ask::Questions::open();
22908        let q = store.get(record.question.as_ref().unwrap()).unwrap();
22909        assert!(
22910            q.choices
22911                .contains(&crate::github_text::USE_REWRITE.to_owned())
22912        );
22913        for label in [
22914            "ORIGINAL",
22915            "FALLBACK",
22916            "REWRITE",
22917            "Rules still failed: body-language",
22918            "truncated",
22919        ] {
22920            assert!(q.detail.contains(label), "{label}");
22921        }
22922        let artifact = std::fs::read_to_string(crate::run::artifact_path(
22923            &runner.state,
22924            &crate::github_text::artifact_name(&record.fingerprint),
22925        ))
22926        .unwrap();
22927        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22928        // Restart from serialized state, then retry after a bad owner title.
22929        runner.state =
22930            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22931        store
22932            .update(&q.id, |q| {
22933                q.say("まだ日本語")?;
22934                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22935            })
22936            .unwrap();
22937        assert!(
22938            runner
22939                .guarded_pr_message(&winner, None, true)
22940                .await
22941                .unwrap()
22942                .is_none()
22943        );
22944        let retry = runner.state.github_text.as_ref().unwrap();
22945        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22946        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22947        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22948        assert!(
22949            q.posting_comparison
22950                .as_ref()
22951                .unwrap()
22952                .matches_detail(&q.detail)
22953        );
22954        store
22955            .update(&q.id, |q| {
22956                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22957            })
22958            .unwrap();
22959        let message = runner
22960            .guarded_pr_message(&winner, None, true)
22961            .await
22962            .unwrap()
22963            .unwrap();
22964        assert_eq!(message.title, rewrite.title);
22965        assert_eq!(message.body, rewrite.body);
22966        assert_eq!(
22967            runner.state.github_text.as_ref().unwrap().choice,
22968            Some(GithubTextChoice::Rewrite)
22969        );
22970        let again = runner
22971            .guarded_pr_message(&winner, None, true)
22972            .await
22973            .unwrap()
22974            .unwrap();
22975        assert_eq!(again.title, message.title);
22976        assert_eq!(again.body, message.body);
22977        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22978        assert!(
22979            !runner
22980                .state
22981                .events
22982                .iter()
22983                .any(|e| e.message.contains(&rewrite.title)
22984                    || e.message.contains("English sentence."))
22985        );
22986    }
22987
22988    #[tokio::test]
22989    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22990        crate::run::pin_test_home();
22991        let dir = tempfile::tempdir().unwrap();
22992        let mut runner = runner_at(RunStatus::Gating);
22993        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22994        runner.state.repo = dir.path().to_owned();
22995        runner.state.candidates[0].worktree = dir.path().to_owned();
22996        seed_public_summary(&mut runner);
22997        let winner = runner.state.candidates[0].clone();
22998        assert!(
22999            runner
23000                .guarded_pr_message(&winner, None, true)
23001                .await
23002                .unwrap()
23003                .is_none()
23004        );
23005        let record = runner.state.github_text.as_ref().unwrap();
23006        assert!(record.rewrite.is_none());
23007        // Records written before the additive field deserialize with no rewrite.
23008        let mut old = serde_json::to_value(record).unwrap();
23009        old.as_object_mut().unwrap().remove("rewrite");
23010        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23011        assert!(old.rewrite.is_none());
23012        let id = record.question.clone().unwrap();
23013        ask::Questions::open()
23014            .update(&id, |q| {
23015                q.choices.push(crate::github_text::USE_REWRITE.into());
23016                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23017            })
23018            .unwrap();
23019        let message = runner
23020            .guarded_pr_message(&winner, None, true)
23021            .await
23022            .unwrap()
23023            .unwrap();
23024        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23025        assert_eq!(
23026            runner.state.github_text.as_ref().unwrap().choice,
23027            Some(GithubTextChoice::Fallback)
23028        );
23029    }
23030
23031    #[tokio::test]
23032    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23033        crate::run::pin_test_home();
23034        let dir = tempfile::tempdir().unwrap();
23035        let mut runner = runner_at(RunStatus::Gating);
23036        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23037        runner.state.repo = dir.path().to_owned();
23038        runner.state.candidates[0].worktree = dir.path().to_owned();
23039        seed_public_summary(&mut runner);
23040        let winner = runner.state.candidates[0].clone();
23041        assert!(
23042            runner
23043                .guarded_pr_message(&winner, None, true)
23044                .await
23045                .unwrap()
23046                .is_none()
23047        );
23048        let store = ask::Questions::open();
23049        let find = |run: &str| {
23050            store
23051                .list()
23052                .into_iter()
23053                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23054                .unwrap()
23055        };
23056        let run = runner.state.id.clone();
23057        let q = find(&run);
23058        let fp = runner
23059            .state
23060            .github_text
23061            .as_ref()
23062            .unwrap()
23063            .fingerprint
23064            .clone();
23065        let name = crate::github_text::artifact_name(&fp);
23066        let path = crate::run::artifact_path(&runner.state, &name);
23067        assert!(q.detail.contains(&path.display().to_string()));
23068        assert!(q.detail.contains("title-language: title"));
23069        let saved = std::fs::read_to_string(&path).expect("artifact written");
23070        assert!(saved.contains("日本語のタイトル"));
23071        assert!(q.cwd.is_none());
23072        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23073        // A bad replacement is refused and asked about once more.
23074        let say = |id: &str, text: &str, choice: &str| {
23075            store
23076                .update(id, |q| {
23077                    q.thread.push(ask::Turn {
23078                        who: ask::Who::Operator,
23079                        body: text.to_owned(),
23080                        at: jiff::Timestamp::now(),
23081                        note: None,
23082                    });
23083                    q.answer(ask::Answer::Choice(choice.to_owned()))
23084                })
23085                .unwrap();
23086        };
23087        say(&q.id, "まだ日本語", "use my text");
23088        assert!(
23089            runner
23090                .guarded_pr_message(&winner, None, true)
23091                .await
23092                .unwrap()
23093                .is_none()
23094        );
23095        let second = find(&run);
23096        assert_ne!(second.id, q.id);
23097        assert!(second.detail.contains("did not pass"));
23098        // The retry still compares the original text; the rejected
23099        // replacement is named by category only.
23100        assert!(second.detail.contains("日本語のタイトル"));
23101        assert!(!second.detail.contains("まだ日本語"));
23102        assert!(second.detail.contains(&path.display().to_string()));
23103        say(&second.id, "fix: retry failed requests", "use my text");
23104        let message = runner
23105            .guarded_pr_message(&winner, None, true)
23106            .await
23107            .unwrap()
23108            .expect("a vetted replacement is posted");
23109        assert_eq!(message.title, "fix: retry failed requests");
23110        assert_eq!(
23111            runner
23112                .state
23113                .github_text
23114                .as_ref()
23115                .unwrap()
23116                .chosen_title
23117                .as_deref(),
23118            Some("fix: retry failed requests")
23119        );
23120        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23121        // Resumed again: the saved decision is applied, nothing is re-asked.
23122        let again = runner
23123            .guarded_pr_message(&winner, None, true)
23124            .await
23125            .unwrap()
23126            .unwrap();
23127        assert_eq!(again.title, message.title);
23128        assert_eq!(
23129            store
23130                .list()
23131                .into_iter()
23132                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23133                .count(),
23134            2
23135        );
23136    }
23137
23138    #[test]
23139    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23140        // What magi itself writes stays English under any configured language,
23141        // so a future localisation of these headings fails here. (The agents'
23142        // own text is also checked by the posting gate.)
23143        let mut state = state_with_summary(
23144            "add retries",
23145            "TITLE: fix(web): batch reads\n- reads run.json once",
23146        );
23147        state.config.graph.language = "ja".to_owned();
23148        let m = pr_message(&state, 'A');
23149        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23150        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23151
23152        // The task is the operator's own text: it goes in untouched, and the
23153        // fallback title (no summary) may be in its language too.
23154        let task = "今回やってほしいこと: results projector を直す";
23155        let mut state = state_with_summary(task, "- no title line");
23156        state.config.graph.language = "ja".to_owned();
23157        let m = pr_message(&state, 'A');
23158        assert_eq!(
23159            m.title,
23160            format!("chore: land candidate A of run {}", state.id)
23161        );
23162        assert!(
23163            m.body.contains(&format!(
23164                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23165            )),
23166            "{}",
23167            m.body
23168        );
23169    }
23170
23171    #[test]
23172    fn pr_message_scrubs_home_paths_and_addresses() {
23173        let state = state_with_summary(
23174            "fix it in /Users/someone/src/x",
23175            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23176        );
23177        let m = pr_message(&state, 'A');
23178        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23179            assert!(!m.body.contains(leak), "{}", m.body);
23180        }
23181        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23182    }
23183
23184    #[test]
23185    fn pr_message_survives_a_task_that_closes_details() {
23186        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23187        let m = pr_message(&state, 'A');
23188        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23189    }
23190
23191    #[test]
23192    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23193        let cmd = manual_merge_command(
23194            MergeStyle::Squash,
23195            Path::new("/repo"),
23196            "b",
23197            "fix: \"quoted\" $(x) `y`\n\nbody",
23198        );
23199        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23200    }
23201
23202    #[test]
23203    fn manual_merge_command_matches_the_configured_style() {
23204        let repo = Path::new("/repo");
23205        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23206
23207        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23208        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23209
23210        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23211        assert_eq!(
23212            squash,
23213            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23214             \"Merge magi run 0832 (candidate A)\""
23215        );
23216
23217        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23218        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23219    }
23220
23221    #[test]
23222    fn a_nudge_gets_a_quarter_of_the_budget() {
23223        // The judge and implement budgets magi ships with.
23224        assert_eq!(retry_budget(secs(1200), true), secs(300));
23225        assert_eq!(retry_budget(secs(3600), true), secs(900));
23226    }
23227
23228    #[test]
23229    fn a_resent_prompt_keeps_the_whole_budget() {
23230        // The seat kept no context, so the retry is the original job again and
23231        // shortening it would only guarantee a second failure.
23232        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23233        assert_eq!(retry_budget(secs(60), false), secs(60));
23234    }
23235
23236    #[test]
23237    fn the_floor_never_exceeds_the_original_budget() {
23238        // A short configured timeout must not be *raised* by the floor: the
23239        // operator asked for a bound, and a retry may not outlast the attempt
23240        // it is retrying.
23241        assert_eq!(retry_budget(secs(60), true), secs(60));
23242        assert_eq!(retry_budget(secs(480), true), secs(120));
23243        assert_eq!(retry_budget(secs(0), true), secs(0));
23244    }
23245
23246    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23247        agent::CommandEvidence {
23248            id: "item1".to_owned(),
23249            description: "cargo test".to_owned(),
23250            exit_code,
23251            result_summary: String::new(),
23252            source: "codex".to_owned(),
23253        }
23254    }
23255
23256    #[test]
23257    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23258        // No evidence is not the same fact as unconfirmed evidence: a
23259        // backend with no adapter, or a reply that ran no commands at all,
23260        // must not be misread as carrying a dangling job.
23261        assert!(!has_unconfirmed_command(&[]));
23262    }
23263
23264    #[test]
23265    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23266        // Deliberately not a check on the exit code's *value*: a fixer
23267        // legitimately runs something that fails mid-iteration before it
23268        // succeeds, and that must never by itself reopen a valid report.
23269        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23270        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23271        assert!(!has_unconfirmed_command(&[
23272            evidence(Some(0)),
23273            evidence(Some(101))
23274        ]));
23275    }
23276
23277    #[test]
23278    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23279        assert!(has_unconfirmed_command(&[
23280            evidence(Some(0)),
23281            evidence(None)
23282        ]));
23283    }
23284
23285    #[test]
23286    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23287        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23288        assert_eq!(
23289            verified_noop_claim(true, &[], text).as_deref(),
23290            Some("already fixed by b32cfc4, on main.")
23291        );
23292    }
23293
23294    #[test]
23295    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23296        // A timeout or a bad exit code reads as the ordinary loss it is,
23297        // whatever the reply's own prose claims.
23298        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23299        assert!(verified_noop_claim(false, &[], text).is_none());
23300    }
23301
23302    #[test]
23303    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23304        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23305        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23306        // A confirmed command alongside the marker is fine.
23307        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23308    }
23309
23310    #[test]
23311    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23312        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23313    }
23314
23315    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23316    /// pair, in order, labelled A, B, C, ...
23317    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23318        runner.state.candidates = shape
23319            .iter()
23320            .enumerate()
23321            .map(|(i, &(empty, verified))| Candidate {
23322                index: i,
23323                label: (b'A' + i as u8) as char,
23324                agent: "sonnet".to_owned(),
23325                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23326                worktree: PathBuf::from(format!("/wt/{i}")),
23327                summary: String::new(),
23328                stat: String::new(),
23329                files: 0,
23330                commits: 0,
23331                empty,
23332                failed: None,
23333                verified_noop: verified.map(str::to_owned),
23334                duration_ms: 0,
23335                folded: false,
23336            })
23337            .collect();
23338    }
23339
23340    #[test]
23341    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23342        ask_test_home();
23343        let mut runner = runner_at(RunStatus::Implementing);
23344        set_candidates(
23345            &mut runner,
23346            &[
23347                (true, Some("already on main at b32cfc4")),
23348                (true, Some("same fix, see the existing test")),
23349            ],
23350        );
23351
23352        runner
23353            .after_implement()
23354            .expect("a verified no-op is not an error");
23355
23356        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23357    }
23358
23359    #[test]
23360    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23361        ask_test_home();
23362        let mut runner = runner_at(RunStatus::Implementing);
23363        // Candidate A declares a verified no-op; candidate B simply wrote
23364        // nothing and said nothing about why. One candidate's claim is not
23365        // the whole run's agreement.
23366        set_candidates(
23367            &mut runner,
23368            &[(true, Some("already on main at b32cfc4")), (true, None)],
23369        );
23370
23371        let err = runner
23372            .after_implement()
23373            .expect_err("an unverified empty candidate must still fail the run");
23374
23375        assert!(
23376            err.to_string().contains("no candidate produced a change"),
23377            "{err}"
23378        );
23379        assert_eq!(runner.state.status, RunStatus::Failed);
23380    }
23381
23382    #[test]
23383    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23384        ask_test_home();
23385        let mut runner = runner_at(RunStatus::Implementing);
23386        set_candidates(&mut runner, &[(true, None), (true, None)]);
23387
23388        let err = runner
23389            .after_implement()
23390            .expect_err("no candidate declared anything; this is an ordinary failure");
23391
23392        assert!(
23393            err.to_string().contains("no candidate produced a change"),
23394            "{err}"
23395        );
23396        assert_eq!(runner.state.status, RunStatus::Failed);
23397    }
23398}
23399
23400/// A graft rollback whose restore could not be confirmed (see
23401/// [`Runner::graft`]).
23402#[derive(Debug)]
23403struct GraftRestoreFailed(String);
23404
23405impl std::fmt::Display for GraftRestoreFailed {
23406    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23407        write!(
23408            f,
23409            "could not restore the winner to its pre-graft commit: {}",
23410            self.0
23411        )
23412    }
23413}
23414
23415impl std::error::Error for GraftRestoreFailed {}
23416
23417/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23418/// not the agent: they never draw on an agent's own timeout, and running out of
23419/// them never reads as an agent failure.
23420#[cfg(test)]
23421mod resource_wait_tests {
23422    use super::*;
23423    use std::collections::BTreeMap;
23424
23425    fn sh(id: &str, script: &str) -> AgentSpec {
23426        AgentSpec {
23427            id: id.to_owned(),
23428            kind: crate::config::AgentKind::Command,
23429            model: None,
23430            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23431            extra_args: Vec::new(),
23432            env: BTreeMap::new(),
23433            prompt_delivery: None,
23434            max_turn_secs: None,
23435        }
23436    }
23437
23438    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23439        SeatJob {
23440            seat: SeatState::new("impl-A", &spec.id, 1),
23441            spec,
23442            cwd: dir.to_path_buf(),
23443            prompt: "do it".to_owned(),
23444            timeout,
23445            allow_write: true,
23446            sessions: false,
23447            artifacts: dir.join("art"),
23448            stem: "impl-A".to_owned(),
23449            handover: None,
23450        }
23451    }
23452
23453    fn state(resource_wait_secs: u64) -> RunState {
23454        let mut config = Config::default();
23455        config.graph.timeout_resource_wait = resource_wait_secs;
23456        RunState::new(
23457            PathBuf::from("/repo"),
23458            "main".to_owned(),
23459            "abc1234".to_owned(),
23460            "task".to_owned(),
23461            config,
23462        )
23463    }
23464
23465    fn hold(cache: &Path) -> crate::cache::Guard {
23466        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23467        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23468            crate::cache::AcquireOutcome::Acquired(g) => g,
23469            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23470        }
23471    }
23472
23473    async fn run(
23474        jobs: Vec<SeatJob>,
23475        sem: Arc<Semaphore>,
23476        cache: Option<&Path>,
23477        st: &mut RunState,
23478    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23479        let prompts = Prompts::default();
23480        let ctx = WaveCtx {
23481            run: "r",
23482            node: "implement",
23483            prompts: &prompts,
23484            cache,
23485            round: None,
23486            carry_seats: false,
23487            pauses: None,
23488        };
23489        wave(jobs, sem, &ctx, st, 0).await
23490    }
23491
23492    #[tokio::test]
23493    async fn verification_cache_wait_preserves_the_command_budget() {
23494        crate::run::pin_test_home();
23495        let tmp = tempfile::tempdir().unwrap();
23496        let cache = tmp.path().join("target");
23497        let guard = hold(&cache);
23498        let release = tokio::spawn(async move {
23499            tokio::time::sleep(Duration::from_millis(100)).await;
23500            guard.release();
23501        });
23502        let mut st = state(10);
23503        let budget = Duration::from_millis(50);
23504        let wait = repository_wait(&st, None);
23505        let (outcomes, _) = with_cache_lease(
23506            &mut st,
23507            Some(&cache),
23508            "gate",
23509            "gate",
23510            tmp.path(),
23511            "abc1234",
23512            budget,
23513            "test gate",
23514            &wait,
23515            |_, available| async move {
23516                assert_eq!(
23517                    available, budget,
23518                    "contention must not shorten verification"
23519                );
23520                (Vec::new(), false, Vec::new())
23521            },
23522        )
23523        .await
23524        .unwrap();
23525        release.await.unwrap();
23526        assert!(outcomes.is_empty(), "{outcomes:?}");
23527        assert!(
23528            st.events
23529                .iter()
23530                .any(|event| event.message.contains("waiting for"))
23531        );
23532        assert!(st.resource_wait.is_none());
23533        // The verification body no longer owns the lease after returning.
23534        hold(&cache).release();
23535    }
23536
23537    #[tokio::test]
23538    async fn verification_cache_wait_expires_without_starting_commands() {
23539        crate::run::pin_test_home();
23540        let tmp = tempfile::tempdir().unwrap();
23541        let cache = tmp.path().join("target");
23542        let _held = hold(&cache);
23543        let mut st = state(0);
23544        let wait = repository_wait(&st, None);
23545        let (outcomes, _) = with_cache_lease(
23546            &mut st,
23547            Some(&cache),
23548            "gate",
23549            "gate",
23550            tmp.path(),
23551            "abc1234",
23552            Duration::from_secs(60),
23553            "test gate",
23554            &wait,
23555            |_, _| async { panic!("commands must not start without the cache lease") },
23556        )
23557        .await
23558        .unwrap();
23559        assert_eq!(outcomes.len(), 1);
23560        assert!(outcomes[0].resource_blocked);
23561        assert!(st.resource_wait.is_none());
23562        assert!(matches!(
23563            st.resource_block.unwrap().cause,
23564            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23565        ));
23566    }
23567
23568    #[tokio::test]
23569    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23570        crate::run::pin_test_home();
23571        let tmp = tempfile::tempdir().unwrap();
23572        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23573        let cache = tmp.path().join("cache");
23574        let mut state = state(21600);
23575        let wait = repository_wait(&state, None);
23576        let (outcomes, _) = with_cache_lease(
23577            &mut state,
23578            Some(&cache),
23579            "gate",
23580            "gate",
23581            tmp.path(),
23582            "head",
23583            Duration::from_secs(60),
23584            "gate",
23585            &wait,
23586            |_, _| async { panic!("freshness failure must not start verification") },
23587        )
23588        .await
23589        .unwrap();
23590        assert_eq!(outcomes.len(), 1);
23591        assert!(outcomes[0].resource_blocked);
23592        let block = state.resource_block.unwrap();
23593        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23594        assert!(!block.description().contains("timed out"));
23595    }
23596
23597    #[tokio::test]
23598    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23599        crate::run::pin_test_home();
23600        let tmp = tempfile::tempdir().unwrap();
23601        let cache = tmp.path().join("cache");
23602        let _held = hold(&cache);
23603        let mut state = state(21600);
23604        let pause = Pause::new();
23605        let interrupt = Pause::new();
23606        let cause = ParkCause::Operator;
23607        pause.park_for(cause.clone());
23608        let prompts = Prompts::default();
23609        let ctx = WaveCtx {
23610            run: "test",
23611            node: "fix",
23612            prompts: &prompts,
23613            cache: Some(&cache),
23614            round: None,
23615            carry_seats: false,
23616            pauses: Some((&pause, &interrupt)),
23617        };
23618        let result = wave(
23619            vec![job(
23620                sh("writer", "exit 99"),
23621                tmp.path(),
23622                Duration::from_secs(1),
23623            )],
23624            Arc::new(Semaphore::new(1)),
23625            &ctx,
23626            &mut state,
23627            0,
23628        )
23629        .await;
23630        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23631        assert_eq!(result[0].1.turns, 0);
23632        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23633        assert_eq!(
23634            state.resource_block.unwrap().cause,
23635            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23636        );
23637        assert!(state.active.is_empty());
23638    }
23639
23640    #[tokio::test]
23641    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23642        crate::run::pin_test_home();
23643        let tmp = tempfile::tempdir().unwrap();
23644        let cache = tmp.path().join("target-a");
23645        let guard = hold(&cache);
23646        // Held for longer than the agent's whole 1s budget.
23647        tokio::spawn(async move {
23648            tokio::time::sleep(Duration::from_millis(1500)).await;
23649            guard.release();
23650        });
23651        let mut st = state(60);
23652        let results = run(
23653            vec![job(
23654                sh("alpha", "echo ok"),
23655                tmp.path(),
23656                Duration::from_secs(1),
23657            )],
23658            Arc::new(Semaphore::new(1)),
23659            Some(&cache),
23660            &mut st,
23661        )
23662        .await;
23663        assert!(
23664            matches!(results[0].2, AgentOutcome::Ok(_)),
23665            "the wait must not be charged to the agent"
23666        );
23667        assert!(st.resource_block.is_none());
23668        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23669        assert!(st.resource_wait.is_none() && st.active.is_empty());
23670    }
23671
23672    #[tokio::test]
23673    #[cfg(unix)]
23674    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23675        use std::os::fd::AsRawFd as _;
23676        crate::run::pin_test_home();
23677        let tmp = tempfile::tempdir().unwrap();
23678        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23679        let common = tmp.path().join(".git");
23680        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23681        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23682        assert_eq!(
23683            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23684            0
23685        );
23686        let mut st = state(1);
23687        st.config.coordination = crate::config::test_coordination();
23688        let mut reader = job(
23689            sh("reader", "echo must-not-run"),
23690            tmp.path(),
23691            Duration::from_secs(30),
23692        );
23693        reader.allow_write = false;
23694        let results = run(
23695            vec![reader.clone()],
23696            Arc::new(Semaphore::new(1)),
23697            None,
23698            &mut st,
23699        )
23700        .await;
23701        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23702        assert_eq!(results[0].1.turns, 0);
23703        assert!(FailClass::of(&results[0].2).is_none());
23704        assert_eq!(
23705            st.resource_block.as_ref().unwrap().kind,
23706            "repository coordination"
23707        );
23708        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23709        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23710        drop(external);
23711        let editor = crate::repo_lock::SeatLease::for_checkout(
23712            tmp.path(),
23713            true,
23714            crate::config::test_coordination().as_ref(),
23715        )
23716        .await
23717        .unwrap();
23718        let pause = Pause::new();
23719        let interrupt = Pause::new();
23720        let prompts = Prompts::default();
23721        let ctx = WaveCtx {
23722            run: "r",
23723            node: "review",
23724            prompts: &prompts,
23725            cache: None,
23726            round: None,
23727            carry_seats: false,
23728            pauses: Some((&pause, &interrupt)),
23729        };
23730        let (results, ()) = tokio::join!(
23731            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23732            async {
23733                tokio::time::sleep(Duration::from_millis(50)).await;
23734                pause.park();
23735            }
23736        );
23737        assert!(
23738            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23739        );
23740        assert_eq!(results[0].1.turns, 0);
23741        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23742        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23743        drop(editor);
23744    }
23745
23746    #[tokio::test]
23747    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23748        crate::run::pin_test_home();
23749        let tmp = tempfile::tempdir().unwrap();
23750        let cache = tmp.path().join("target-b");
23751        let _held = hold(&cache);
23752        let mut st = state(1);
23753        let results = run(
23754            vec![job(
23755                sh("alpha", "echo ok"),
23756                tmp.path(),
23757                Duration::from_secs(1),
23758            )],
23759            Arc::new(Semaphore::new(1)),
23760            Some(&cache),
23761            &mut st,
23762        )
23763        .await;
23764        let (_, seat, out) = &results[0];
23765        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23766        assert!(wave_blocked(&results));
23767        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23768        assert_eq!(seat.turns, 0, "nothing was asked");
23769        assert!(
23770            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23771            "no agent was spawned"
23772        );
23773        assert!(FailClass::of(out).is_none(), "not a failure class");
23774        let block = st.resource_block.as_ref().expect("recorded for the resume");
23775        assert_eq!(block.kind, "shared build cache");
23776        assert!(matches!(
23777            block.cause,
23778            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23779        ));
23780        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23781        assert!(st.active.is_empty());
23782    }
23783
23784    #[tokio::test]
23785    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23786        crate::run::pin_test_home();
23787        let tmp = tempfile::tempdir().unwrap();
23788        let mut st = state(1);
23789        let results = run(
23790            vec![job(
23791                sh("alpha", "echo ok"),
23792                tmp.path(),
23793                Duration::from_secs(1),
23794            )],
23795            Arc::new(Semaphore::new(0)),
23796            None,
23797            &mut st,
23798        )
23799        .await;
23800        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23801        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23802        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23803    }
23804
23805    #[tokio::test]
23806    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23807        crate::run::pin_test_home();
23808        let tmp = tempfile::tempdir().unwrap();
23809        let mut st = state(60);
23810        let results = run(
23811            vec![job(
23812                sh("alpha", "sleep 30"),
23813                tmp.path(),
23814                Duration::from_secs(1),
23815            )],
23816            Arc::new(Semaphore::new(1)),
23817            None,
23818            &mut st,
23819        )
23820        .await;
23821        let out = &results[0].2;
23822        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23823        let class = FailClass::of(out).expect("a failure");
23824        assert_eq!(class, FailClass::Timeout);
23825        assert!(should_hand_over(None, &class), "existing handover policy");
23826        assert!(st.resource_block.is_none());
23827    }
23828}