Skip to main content

magi/
graph.rs

1//! The competition graph.
2//!
3//! ```text
4//! prep ──► implement ×N ──► judge ×M (blind) ──► split? ──► deliberate ──► vote (private)
5//!                                                   │                          │
6//!                                                   └──── unanimous ───────────┤
7//!                                                                              ▼
8//!   merge ◄── gate ◄── review ×R + E2E, fix, repeat ◄── fold losers ◄──────── tally
9//! ```
10//!
11//! Every node persists before the next one starts, so a run can be resumed
12//! after a crash, a rate limit, or a reboot without re-spending the work that
13//! already landed.
14//!
15//! The design decision that matters most is *where the facilitator lives*.
16//! There is no moderator agent: magi assigns the labels, decides the
17//! presentation order, relays the transcript, and collects the final votes
18//! one-to-one. A moderator that never learns an author cannot leak one.
19use std::collections::{BTreeMap, BTreeSet};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, Ordering};
22use std::sync::{Arc, Mutex};
23use std::time::{Duration, Instant};
24
25use anyhow::{Context as _, Result, bail};
26use jiff::Timestamp;
27use tokio::sync::Semaphore;
28
29use crate::advise;
30use crate::agent::{self, AgentOutput, Invocation, SeatState};
31use crate::ask;
32use crate::blind;
33use crate::bump;
34use crate::config::{
35    AgentSpec, Config, IncompleteReviewPolicy, LeakPolicy, MergeMode, MergeStyle, Prompts,
36    ResolvedRoles,
37};
38use crate::fixer;
39use crate::git;
40use crate::graft;
41use crate::land;
42use crate::proc::Quiet as _;
43use crate::prompt::{
44    self, CandidateView, Lens, ReviewPatch, ReviewReconsiderCtx, ReviewSeatReport, Turn,
45};
46use crate::queue;
47#[cfg(test)]
48use crate::refs;
49use crate::run::{
50    BaseSync, Candidate, CommandOutcome, ContinuationOutcome, ContinuationRecord,
51    DeliberationRound, DeliberationTurn, E2eStatus, FailClass, FixRecord, GateFixRecord,
52    GithubTextChoice, GraftOutcome, GraftRecord, Handover, JobRecord, JobStatus, Judgement,
53    MergeOutcome, OperatorFixFinding, OperatorFixOutcome, OperatorFixRequest, Origin, ParkCause,
54    QuotaLoss, ReviewRecord, ReviewRevoteRecord, ReviewRound, RunState, RunStatus, SeatHistory,
55    Tally, VoteRecord, summarize_verify, write_artifact,
56};
57use crate::verdict::{
58    self, FinalVote, Finding, FixReport, Position, Proposal, Ranking, Review, ReviewRevote,
59    ReviewVote, Severity,
60};
61use crate::worktree_setup;
62
63/// Bytes of a failing command's output kept in an event, so the reason a run
64/// stopped is readable from the report without opening `run.json`.
65const EVENT_OUTPUT_TAIL: usize = 2_000;
66
67/// How often [`wait_for_timed_out_children_to_die`] re-checks a timed-out
68/// command's pid before releasing the build cache's lease.
69const LEASE_RELEASE_POLL: Duration = Duration::from_secs(1);
70
71/// The most [`wait_for_timed_out_children_to_die`] will wait for a timed-out
72/// command's pid to actually exit before giving up and releasing anyway.
73///
74/// A timeout means the process was asked to die (`kill_on_drop`,
75/// `start_kill`), not that it already has — on Windows in particular that can
76/// take a moment, the same reason `agent`'s own `PIPE_GRACE` exists. Releasing
77/// the instant the command returns would let the very next acquirer (this
78/// run's own next round, another run's verification, the janitor's prune)
79/// start touching the same directory while it might still be writing to it,
80/// so this polls the actual pid — real confirmation, not a fixed guess —
81/// until it is gone or this ceiling is reached. It is still not full
82/// process-tree reaping: a grandchild the timed-out process spawned and that
83/// outlives it independently is invisible to a pid check, and continuing to
84/// observe and collect *that* stays a different piece of work with its own
85/// owner. Set generously because the common case returns early the moment
86/// the pid is confirmed gone, not because every timeout pays this in full.
87const LEASE_RELEASE_MAX_WAIT: Duration = Duration::from_secs(30);
88
89/// Consecutive review rounds with no tree progress (see
90/// [`crate::run::ReviewRound::progressed`]) before `review_loop` hands off
91/// instead of spending the rest of the round budget.
92///
93/// Not 1: a single non-progressing round is not yet a pattern — a fixer that
94/// legitimately finds nothing left to change (its previous round's fix already
95/// covered it, and this round's reviewers re-raised only nits) looks the same
96/// as one that is spinning, for exactly one round. Two in a row is where the
97/// two stop being distinguishable, and a review round on this workload has
98/// been measured at 30-45 minutes of reviewer-plus-fixer agent time, so a
99/// third attempt at a tree that has not moved twice running is pure cost.
100/// This does not touch `review_rounds` itself, which stays the operator's
101/// call.
102pub(crate) const STAGNANT_LIMIT: usize = 2;
103
104/// How many times [`Runner::sync_to_base`] will re-land the winner's tree on
105/// a base that moved before giving up and leaving the run `Blocked` for a
106/// person.
107///
108/// Mirrors `land::Step::Rebase`'s budget and the reasoning behind it: a base
109/// that keeps moving faster than a run can catch it is not something more
110/// rebasing fixes, it is a person's call. Not the same *number as*
111/// `land_rounds` - this budget is spent before a pull request exists, land's
112/// after - but bounded for the identical reason, so it uses the same
113/// default. Counted across both call sites in [`Runner::finish_after_tally`]
114/// (once before review, once before the gate), because either one finding
115/// the base still moving is the same signal.
116pub(crate) const BASE_SYNC_ROUNDS: usize = 4;
117
118/// How many times [`Runner::continue_fix_report`] will resume the fixer's own
119/// seat when its CLI turn ended cleanly — usable, non-empty, exit 0 — but the
120/// reply held no [`FixReport`].
121///
122/// The shape this recovers: run 20260912-114326-d3b8's fix-2 came back
123/// `subtype=success`/`is_error=false`/`stop_reason=end_turn` with the reply
124/// "I'll pause here until the `cargo make check` background run reports
125/// back." — a CLI turn that ended cleanly while the fixer's own job had not.
126/// No `FixReport` was ever collected from that seat, and the run moved on to
127/// the next review round regardless.
128///
129/// Bounded independently of `review_rounds` and `graph.retries`: this
130/// recovers one seat's missing report mid-round, not a new round of review or
131/// an ordinary parse retry, and must not itself become the unbounded wait the
132/// rest of this module exists to avoid.
133const MAX_FIX_CONTINUATIONS: usize = 2;
134
135/// One queued agent invocation.
136///
137/// `Clone` so a node can keep the jobs it sent and re-send one: a seat whose
138/// CLI hung up on its own stream is asked again from the same job rather than
139/// rebuilt from scratch. See [`Runner::resume_undelivered`].
140#[derive(Clone)]
141struct SeatJob {
142    spec: AgentSpec,
143    seat: SeatState,
144    cwd: PathBuf,
145    prompt: String,
146    timeout: Duration,
147    allow_write: bool,
148    sessions: bool,
149    artifacts: PathBuf,
150    stem: String,
151    /// The prompt for a seat that has been handed to another roster agent
152    /// (a fresh session): everything the original seat would have
153    /// remembered. `None` when `prompt` already carries it, as the first
154    /// ranking and the implement prompt do. Never a resume-style prompt.
155    handover: Option<String>,
156}
157
158/// How the graph reads one agent invocation.
159///
160/// Quota is split out from an ordinary failure on purpose: a rate-limited call
161/// is known to fail again if retried now, so the retry loop must not spend an
162/// attempt on it. `Dropped` is split out for the opposite reason: unlike
163/// `Failed`, it is worth re-asking, and unlike `Ok`, its text is the CLI's raw
164/// error JSON, never the agent's answer — a caller that matched only
165/// `Ok`/`Quota`/`Failed` before `Dropped` existed must be updated rather than
166/// left to read that JSON as if it were usable output. `resume_undelivered`
167/// is the only caller that acts on it; everywhere else it is reported like an
168/// ordinary failure.
169enum AgentOutcome {
170    /// A usable output.
171    Ok(AgentOutput),
172    /// The CLI ran out of quota / rate limit. Retrying now is pointless.
173    Quota(AgentOutput),
174    /// The CLI hung up on its own stream after billed work. See
175    /// [`agent::AgentOutput::work_undelivered`].
176    Dropped(AgentOutput),
177    /// Any other failure: a timeout, a bad exit code, an empty reply.
178    Failed(String),
179    /// A resource prevented admission or interrupted execution. Not an agent
180    /// failure: never advance a fallback chain or spend an attempt on it.
181    /// Post-spawn interruptions carry a typed error into wave, which records
182    /// implementation preservation before returning this outcome. This outcome
183    /// alone is not proof that no source changes were made.
184    /// [`RunState::resource_block`] says why.
185    ResourceBlocked(String),
186}
187
188/// A request to park the run at its next node boundary.
189///
190/// Cloning is how the request travels: the loop keeps one handle and hands a
191/// clone to each [`Runner`], and every clone points at the same flag. There
192/// is no channel because there is nothing to send - the only message is
193/// "park", it is idempotent, and a flag cannot be missed by a receiver that
194/// was not listening yet.
195///
196/// The boundary is what makes this cheap. Every node writes the run's state
197/// before the next one starts, and every node skips what is already recorded:
198/// `prep` returns early once candidates exist, `implement` asks only the seats
199/// with nothing on disk, `judge` returns early once judgements exist. So a
200/// parked run resumes into exactly the node it stopped before, and no agent
201/// work is thrown away. Killing the process mid-node, by contrast, loses
202/// whatever the seats in flight had not yet written - which for an implement
203/// wave is an hour of paid work.
204///
205/// A [`Runner`] watches two independent handles of this type - see
206/// [`Runner::on_pause`] and [`Runner::watch_interrupt`] - never one shared
207/// between them. `magi serve`'s own shutdown (`Stop::park`) hands out one
208/// clone covering the whole daemon's lifetime and is never asked to un-park,
209/// which is correct exactly because nothing is dispatched after it fires.
210/// `magi serve`'s interrupt scheduler needs the opposite lifetime - a run
211/// that parks for an interrupted task must go on to run other tasks
212/// afterward - so it mints a fresh, unshared [`Pause`] per run instead of
213/// reusing the daemon-wide one.
214#[derive(Debug, Clone, Default)]
215pub struct Pause(Arc<AtomicBool>, Arc<Mutex<Option<ParkCause>>>);
216
217impl Pause {
218    /// A pause nobody has asked for yet.
219    #[must_use]
220    pub fn new() -> Self {
221        Self::default()
222    }
223
224    /// Ask the run to park at its next node boundary. Idempotent.
225    pub fn park(&self) {
226        self.0.store(true, Ordering::SeqCst);
227    }
228
229    /// Same as [`Pause::park`], but records why, for [`Runner::park_here`] to
230    /// fold into the run's own `park` event - so an operator reading the run
231    /// later knows this was a deliberate interrupt rather than a shutdown or
232    /// a binary swap. The first reason recorded wins; a park already in
233    /// flight is not relabelled by a second, unrelated request.
234    pub fn park_because(&self, reason: impl Into<String>) {
235        self.park_for(ParkCause::Interrupt {
236            reason: reason.into(),
237        });
238    }
239
240    /// [`Pause::park`], recording the structured cause. The first cause
241    /// recorded wins, like [`Pause::park_because`].
242    pub fn park_for(&self, cause: ParkCause) {
243        let mut reason_guard = self
244            .1
245            .lock()
246            .unwrap_or_else(std::sync::PoisonError::into_inner);
247        if reason_guard.is_none() {
248            *reason_guard = Some(cause);
249        }
250        drop(reason_guard);
251        self.park();
252    }
253
254    /// Has a park been asked for?
255    #[must_use]
256    pub fn parked(&self) -> bool {
257        self.0.load(Ordering::SeqCst)
258    }
259
260    /// Why the park was asked for, when the caller used [`Pause::park_because`].
261    #[must_use]
262    pub fn reason(&self) -> Option<String> {
263        self.cause().map(|c| c.label())
264    }
265
266    /// The structured cause, when one was recorded.
267    #[must_use]
268    pub fn cause(&self) -> Option<ParkCause> {
269        self.1
270            .lock()
271            .unwrap_or_else(std::sync::PoisonError::into_inner)
272            .clone()
273    }
274}
275
276/// Drives one run.
277pub struct Runner {
278    /// Run state; public so the CLI can report on it.
279    pub state: RunState,
280    roles: ResolvedRoles,
281    sem: Arc<Semaphore>,
282    /// Set when the daemon's own shutdown (Ctrl-C, a binary swap) wants the
283    /// run parked at its next node boundary. See [`Pause`]'s own doc for why
284    /// this is never the same handle as `interrupt`.
285    pause: Pause,
286    /// Set when `magi serve`'s interrupt scheduler wants this specific run
287    /// parked at its next node boundary, to let a task marked
288    /// [`crate::queue::Task::interrupt`] run alone before this one carries
289    /// on. Unlike `pause`, a fresh, unshared handle per run - see
290    /// [`Runner::watch_interrupt`].
291    interrupt: Pause,
292}
293
294/// Where the branch's own commits start: its merge base with the base branch
295/// as the remote has it now, else with the recorded `base_commit`. A branch
296/// rebased onto a base that moved past `base_commit` would otherwise count
297/// the base's commits as its own under `base_commit..branch`.
298async fn review_base(
299    repo: &Path,
300    remote: &str,
301    base_branch: &str,
302    base_commit: &str,
303    branch: &str,
304) -> String {
305    review_base_checked(repo, remote, base_branch, base_commit, branch)
306        .await
307        .0
308}
309
310/// [`review_base`] plus whether the base was read from a freshly fetched
311/// tracking ref. A failed fetch still uses whatever tracking ref exists (it is
312/// never older than `base_commit`'s view of the base), but the answer is then
313/// not trusted to rewrite a pull request's title.
314async fn review_base_checked(
315    repo: &Path,
316    remote: &str,
317    base_branch: &str,
318    base_commit: &str,
319    branch: &str,
320) -> (String, bool) {
321    let tracking = format!("{remote}/{base_branch}");
322    let fresh = matches!(git::fetch(repo, remote, base_branch).await, Ok(o) if o.ok());
323    if git::rev_exists(repo, &tracking).await
324        && let Ok(mb) = git::merge_base(repo, &tracking, branch).await
325        && !mb.is_empty()
326    {
327        return (mb, fresh);
328    }
329    let mb = git::merge_base(repo, base_commit, branch)
330        .await
331        .ok()
332        .filter(|mb| !mb.is_empty())
333        .unwrap_or_else(|| base_commit.to_owned());
334    (mb, false)
335}
336
337/// Recompute `reviewed_commits` from the branch's own commits. Left as it was
338/// when git cannot say or finds nothing: a stale list is better than a wrong
339/// or empty one.
340pub(crate) async fn refresh_reviewed_commits(state: &mut RunState, branch: &str) {
341    if !is_review_run(state) {
342        return;
343    }
344    let base = review_base(
345        &state.repo,
346        &state.config.merge.remote,
347        &state.base_branch,
348        &state.base_commit,
349        branch,
350    )
351    .await;
352    if let Ok(subjects) = git::subjects(&state.repo, &base, branch).await
353        && !subjects.is_empty()
354        && state.reviewed_commits.as_ref() != Some(&subjects)
355    {
356        state.reviewed_commits = Some(subjects);
357        state.save().ok();
358    }
359}
360
361/// Subjects of the base's commits between the recorded start and the branch's
362/// merge base: what a stale `base_commit..branch` would have mistaken for the
363/// branch's own work.
364async fn leaked_subjects(state: &RunState, branch: &str) -> Option<Vec<String>> {
365    let (base, trusted) = review_base_checked(
366        &state.repo,
367        &state.config.merge.remote,
368        &state.base_branch,
369        &state.base_commit,
370        branch,
371    )
372    .await;
373    if !trusted {
374        return None;
375    }
376    git::subjects(&state.repo, &state.base_commit, &base)
377        .await
378        .ok()
379}
380
381/// May an adopted pull request's title be replaced with `computed`? Only when
382/// it is empty, magi's own shape, or a base commit's subject that leaked in;
383/// a title a person wrote stays. Never when `computed` is itself a leak.
384pub(crate) fn should_retitle(current: &str, computed: &str, leaked: &[String]) -> bool {
385    let is_leak = |t: &str| leaked.iter().any(|l| l.trim() == t.trim());
386    if is_leak(computed) {
387        return false;
388    }
389    let cur = current.trim();
390    cur.is_empty()
391        || cur.starts_with(REVIEW_PROMPT_OPENING)
392        || cur.starts_with("chore: land candidate")
393        || cur.starts_with("magi: candidate")
394        || is_leak(cur)
395}
396
397/// The commit a run branches from: the base branch as the remote has it.
398///
399/// Two failures this replaces. A run used to branch off `HEAD` and so refused
400/// to start on a dirty tree, which made `magi serve` decline every task for as
401/// long as the operator had work in progress - most of the time. Branching off
402/// the *local* base branch fixed that and introduced a worse one: `land` merges
403/// the winner on GitHub, nothing updates the local ref, and the next run
404/// branches off a base missing everything the previous runs landed. Two tasks
405/// in a row from a phone would have had the second silently re-implementing
406/// against stale code and opening a pull request that reverted the first.
407///
408/// Only refs move here - no checkout, no local branch, no merge - so it is safe
409/// with uncommitted work in the tree. A machine with no network still starts:
410/// the fetch may fail and the local tip is used with a warning, because
411/// refusing to run offline is a worse failure than running against a base the
412/// operator can see for themselves.
413///
414/// One function, called by both entry points. Two answers to "where does a run
415/// branch from" is the kind of drift nobody notices until a diff is wrong.
416/// Bring the local `branch` in line with `<remote>/<branch>` before a review
417/// checks it out.
418///
419/// `git worktree add <branch>` resolves the *local* ref, and a branch pushed by
420/// anything other than plain `git push` from this checkout (a jj colocated
421/// workspace, another clone) moves only the remote-tracking ref - so the local
422/// one can be a stale placeholder. It moves only when local is behind the remote or is an
423/// empty placeholder that diverged from it; unpushed local work is kept, and a real
424/// divergence is refused rather than guessed at.
425async fn sync_review_branch(repo: &Path, branch: &str, remote: &str, base: &str) -> Result<()> {
426    let tracking = format!("{remote}/{branch}");
427    let fetched = git::fetch(repo, remote, branch).await;
428    let fresh = matches!(&fetched, Ok(o) if o.ok()) && git::rev_exists(repo, &tracking).await;
429    let local_exists = git::branch_exists(repo, branch).await?;
430    if !fresh {
431        if !local_exists {
432            bail!("no branch `{branch}` in {} or on {remote}", repo.display());
433        }
434        tracing::warn!(
435            "could not read {tracking}; reviewing the local `{branch}`, which may be stale"
436        );
437        return Ok(());
438    }
439    let remote_sha = git::rev_parse(repo, &tracking).await?;
440    if !local_exists {
441        git::git(repo, &["branch", branch, &tracking]).await?;
442        return Ok(());
443    }
444    let local_sha = git::rev_parse(repo, &format!("refs/heads/{branch}")).await?;
445    if local_sha == remote_sha || git::is_ancestor(repo, &remote_sha, &local_sha).await {
446        return Ok(());
447    }
448    if !git::is_ancestor(repo, &local_sha, &remote_sha).await {
449        // Diverged. `reconcile` settles it only when it can prove nothing is
450        // lost: a local tip that is the remote's change rebased is pushed over
451        // it (lease pinned to the tip read here), a tip whose every extra
452        // commit is empty is a placeholder the remote's work replaced, and
453        // anything else is two different changes - a question for a person.
454        match crate::reconcile::reconcile(repo, remote, branch, &local_sha, &remote_sha, base)
455            .await?
456        {
457            crate::reconcile::Reconciliation::Pushed => {
458                tracing::warn!(
459                    "local `{branch}` ({}) is {tracking} ({}) rebased; pushed it over",
460                    short(&local_sha),
461                    short(&remote_sha)
462                );
463                return Ok(());
464            }
465            crate::reconcile::Reconciliation::Placeholder => {}
466            crate::reconcile::Reconciliation::Genuine(d) => return Err((*d).into()),
467        }
468    }
469    let out = git::git_raw(repo, &["branch", "-f", branch, &tracking]).await?;
470    if !out.ok() {
471        bail!(
472            "local `{branch}` ({}) is stale against {tracking} ({}) but git will not move it: {}",
473            short(&local_sha),
474            short(&remote_sha),
475            out.stderr
476        );
477    }
478    tracing::warn!(
479        "local `{branch}` was stale: fast-forwarded {} -> {}",
480        short(&local_sha),
481        short(&remote_sha)
482    );
483    Ok(())
484}
485
486async fn resolve_base(
487    repo: &Path,
488    base_branch: &str,
489    remote: &str,
490    coordination: Option<&crate::config::Coordination>,
491) -> Result<crate::run::FetchedBase> {
492    let _lease = crate::repo_lock::SeatLease::operation_with(
493        repo,
494        coordination,
495        &crate::repo_lock::WaitPolicy::current(),
496    )
497    .await?;
498    let tracking = format!("{remote}/{base_branch}");
499    let fetched = git::fetch(repo, remote, base_branch).await;
500    if let Ok(out) = &fetched
501        && out.ok()
502        && git::rev_exists(repo, &tracking).await
503    {
504        return Ok(crate::run::FetchedBase {
505            remote: remote.to_owned(),
506            branch: base_branch.to_owned(),
507            commit: git::rev_parse(repo, &tracking).await?,
508            fetched_at: jiff::Timestamp::now(),
509        });
510    }
511    let why = match &fetched {
512        Ok(out) if !out.ok() => out.stderr.trim().to_owned(),
513        Ok(_) => format!("{remote} has no {base_branch}"),
514        Err(e) => e.to_string(),
515    };
516    // No fallback to the local branch: it may be behind, and branching off an
517    // old commit is the stale-checkout bug this check exists to prevent.
518    bail!(
519        "cannot read {tracking} ({why}); refusing to branch off the local \
520         `{base_branch}`, which may be behind. Fix the remote, or set [merge] \
521         base / remote in magi.toml. This is launcher preparation: check the \
522         launcher's network, Git access and repository coordination before \
523         retrying"
524    )
525}
526
527/// Outcome of the shared fixer chain, before caller-owned stop policy.
528enum FixStep {
529    Resource,
530    Recorded { report_lost: bool },
531}
532
533/// Exclusive claim shared by targeted repairs and repository reassignment.
534/// Held before asynchronous fix preflight and released on every return or panic.
535///
536/// `daemon::is_working_on` only sees a heartbeat-publishing daemon; two
537/// manual `magi fix` invocations against the same run are otherwise
538/// invisible to each other and would race to remove and recreate the same
539/// worktree (see [`Runner::fix_selected`]). The lock file itself is the same
540/// `create_new` shape as `queue::Claim`, but unlike a queued task's lock —
541/// which is only ever reclaimed later, out of band, by
542/// `daemon::sweep_stale_claims` running inside `magi serve`/`magi web` — a
543/// `magi fix` invocation is not necessarily running under either of those, so
544/// nothing would ever sweep a lock a killed or crashed process left behind.
545/// [`Self::acquire`] therefore reclaims a stale lock itself, on the same
546/// conservative PID-liveness policy `sweep_stale_claims` and `cache`'s own
547/// lease use: an unreadable or unparsable pid, or a liveness query the
548/// platform cannot answer, reads as alive and the lock is left in place.
549pub(crate) struct FixClaim {
550    path: PathBuf,
551}
552
553impl FixClaim {
554    pub(crate) fn acquire(dir: &Path) -> Result<Self> {
555        std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
556        let path = dir.join("fix.lock");
557        let retry = crate::lock_file::Retry::new(None);
558        match Self::create(&path, retry) {
559            Ok(claim) => Ok(claim),
560            Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
561                if Self::reclaim_if_dead(&path) {
562                    Self::create(&path, retry).with_context(|| format!("lock {}", path.display()))
563                } else {
564                    bail!(
565                        "another repair or `magi fix` is already running for this run ({} exists)",
566                        path.display()
567                    )
568                }
569            }
570            Err(e) => Err(e).with_context(|| format!("lock {}", path.display())),
571        }
572    }
573
574    fn create(path: &Path, retry: crate::lock_file::Retry) -> std::io::Result<Self> {
575        let mut f = retry.create(path)?;
576        use std::io::Write as _;
577        // Read back by `reclaim_if_dead` on a later, stuck invocation.
578        writeln!(f, "{}", std::process::id())?;
579        Ok(Self {
580            path: path.to_owned(),
581        })
582    }
583
584    /// True if the lock named a process confirmed dead, in which case it was
585    /// also removed. Never true on an unreadable file, an unparsable pid, or
586    /// a liveness query the platform cannot answer — see this type's own doc.
587    fn reclaim_if_dead(path: &Path) -> bool {
588        let dead = std::fs::read_to_string(path)
589            .ok()
590            .and_then(|body| body.trim().parse::<u32>().ok())
591            .is_some_and(|pid| !crate::proc::pid_alive(pid));
592        dead && std::fs::remove_file(path).is_ok()
593    }
594}
595
596impl Drop for FixClaim {
597    fn drop(&mut self) {
598        let _ = std::fs::remove_file(&self.path);
599    }
600}
601
602/// Holds the mutable runner as well as its durable lease: cancellation must
603/// clear both copies of ownership, including state a caller may save later.
604struct DriverExecution<'a> {
605    runner: &'a mut Runner,
606    driver: crate::driver::DriverGuard,
607}
608
609impl Drop for DriverExecution<'_> {
610    fn drop(&mut self) {
611        self.driver.release_local(&mut self.runner.state);
612    }
613}
614
615fn startup_stage(state: &mut RunState, stage: &str) -> Result<()> {
616    if let Some(startup) = &mut state.startup {
617        startup.stage = stage.to_owned();
618    }
619    state.save()
620}
621
622/// Keep a reservation visible as blocked until startup finishes. A killed process
623/// leaves the same explicit incomplete marker; dropping a cancelled future records it.
624struct StartupGuard {
625    id: String,
626    armed: bool,
627}
628
629impl StartupGuard {
630    fn reserve(state: &mut RunState, kind: &str, branch: Option<&str>) -> Result<Self> {
631        state.startup = Some(crate::run::Startup {
632            kind: kind.to_owned(),
633            branch: branch.map(str::to_owned),
634            stage: "reserved".to_owned(),
635            complete: false,
636            failure: None,
637        });
638        state.status = RunStatus::Blocked;
639        state.reserve()?;
640        Ok(Self {
641            id: state.id.clone(),
642            armed: true,
643        })
644    }
645
646    fn stage(&self, state: &mut RunState, stage: &str) -> Result<()> {
647        startup_stage(state, stage)
648    }
649
650    fn finish(mut self, state: &mut RunState, result: Result<()>) -> Result<()> {
651        let result = result.and_then(|()| {
652            state.startup.as_mut().expect("reserved startup").complete = true;
653            state.save()
654        });
655        self.armed = false;
656        match result {
657            Ok(()) => Ok(()),
658            Err(error) => {
659                let startup = state.startup.as_mut().expect("reserved startup");
660                startup.complete = false;
661                startup.failure = Some(format!("{error:#}"));
662                let context = crate::run::StartupFailure {
663                    run: state.id.clone(),
664                    stage: startup.stage.clone(),
665                };
666                // Repository assets may remain. Never infer that a failed startup
667                // is safe to resume or that an earlier owner has been restored.
668                state.status = RunStatus::Failed;
669                state.driver_exited = true;
670                state.event(
671                    "startup-failed",
672                    format!("{context}: {error:#}; retained assets require inspection"),
673                );
674                if let Err(save) = state.save() {
675                    tracing::warn!(
676                        "run {}: could not save terminal startup record: {save:#}; reservation requires reconciliation",
677                        state.id
678                    );
679                }
680                Err(error.context(context))
681            }
682        }
683    }
684}
685
686impl Drop for StartupGuard {
687    fn drop(&mut self) {
688        if self.armed
689            && let Ok(mut state) = RunState::load(&self.id)
690        {
691            state.status = RunStatus::Blocked;
692            state.driver_exited = true;
693            if let Some(startup) = &mut state.startup {
694                startup.failure = Some("startup interrupted; reconciliation required".to_owned());
695            }
696            state.event(
697                "startup-interrupted",
698                "startup did not finish; inspect retained assets before recovery",
699            );
700            if let Err(error) = state.save() {
701                tracing::error!(
702                    "could not record interrupted startup {}: {error:#}",
703                    self.id
704                );
705            }
706        }
707    }
708}
709
710impl Runner {
711    /// A queue-side failure after construction must not strand an unexecuted run.
712    pub(crate) fn fail_startup(&mut self, stage: &str, error: anyhow::Error) -> anyhow::Error {
713        if !self.state.passes.is_empty() || self.state.driver_claimed != Some(false) {
714            return error;
715        }
716        if let Some(startup) = &mut self.state.startup {
717            startup.stage = stage.to_owned();
718        } else {
719            // A resumed run already has execution history: do not rewrite it.
720            return error;
721        }
722        StartupGuard {
723            id: self.state.id.clone(),
724            armed: true,
725        }
726        .finish(&mut self.state, Err(error))
727        .expect_err("startup failed")
728    }
729
730    /// Start a fresh run against `repo`.
731    pub async fn start(
732        repo: &Path,
733        instruction: String,
734        config: Config,
735        origin: Origin,
736    ) -> Result<Self> {
737        Self::start_naming(repo, instruction, "", config, origin).await
738    }
739
740    /// [`Runner::start`] for a queued task. Titles and instructions never
741    /// select implementation sources. The title argument is retained for callers.
742    pub async fn start_naming(
743        repo: &Path,
744        instruction: String,
745        _also_scan: &str,
746        config: Config,
747        origin: Origin,
748    ) -> Result<Self> {
749        let repo = git::toplevel(repo).await?;
750        let missing = agent::missing_programs(&config.agents);
751        if !missing.is_empty() {
752            bail!(
753                "these agent programs are not on PATH: {}. Fix the roster in \
754                 magi.toml or install them.",
755                missing.join(", ")
756            );
757        }
758        let base_branch =
759            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
760                .await?;
761        let fetched_base = resolve_base(
762            &repo,
763            &base_branch,
764            &config.merge.remote,
765            config.coordination.as_ref(),
766        )
767        .await?;
768        let base_commit = fetched_base.commit.clone();
769        let roles = config.resolve_roles()?;
770        let max_parallel = config.graph.max_parallel.max(1);
771        let mut state = RunState::new(repo, base_branch, base_commit, instruction, config);
772        // Capture task identity and repository assignment before the first
773        // save, so publication scope survives crashes and later task moves.
774        state.fetched_base = Some(fetched_base);
775        state.set_origin(origin)?;
776        let startup = StartupGuard::reserve(&mut state, "implementation", None)?;
777        state.status = RunStatus::Prep;
778        state.event("start", format!("run {} created", state.id));
779        startup.finish(&mut state, Ok(()))?;
780        Ok(Self {
781            state,
782            roles,
783            sem: Arc::new(Semaphore::new(max_parallel)),
784            pause: Pause::new(),
785            interrupt: Pause::new(),
786        })
787    }
788
789    /// Open a review-only run against work that already exists on `branch`.
790    ///
791    /// The expensive half of the graph is the implement wave — measured at
792    /// 111 and 134 internal tool-loop turns on this repository, against a
793    /// handful for a judge or a reviewer. The cheap half is worth running on
794    /// hand-written work too, and there was no way to reach it.
795    ///
796    /// No new state and no schema change are needed: a run with **one** viable
797    /// candidate and a tally already decided degrades `execute` to exactly
798    /// review → gate → merge, because `judge` skips a single-candidate field,
799    /// `deliberate` has fewer than two first choices to reconcile, `vote`
800    /// returns early, `tally` is already present and `fold_losers` has no
801    /// losers. Resuming such a run therefore does the right thing as well.
802    pub async fn review(repo: &Path, branch: &str, config: Config, origin: Origin) -> Result<Self> {
803        Self::review_taking_over(repo, branch, config, None, origin).await
804    }
805
806    /// [`Runner::review`] for a queued task's retry: when an earlier attempt
807    /// at the same task still has `branch` checked out, its worktree is
808    /// released first if that is safe (see [`crate::handover`]), and the
809    /// review refuses with the reason if it is not. `None` is a hand-run
810    /// review: it has no earlier attempts, so only a worktree of a dead run
811    /// magi recorded itself can be released.
812    pub async fn review_taking_over(
813        repo: &Path,
814        branch: &str,
815        config: Config,
816        takeover: Option<crate::handover::Takeover>,
817        origin: Origin,
818    ) -> Result<Self> {
819        let repo = git::toplevel(repo).await?;
820        let missing = agent::missing_programs(&config.agents);
821        if !missing.is_empty() {
822            bail!(
823                "these agent programs are not on PATH: {}. Fix the roster in \
824                 magi.toml or install them.",
825                missing.join(", ")
826            );
827        }
828        let base_branch =
829            git::merge_base_branch(&repo, &config.merge.remote, config.merge.base.as_deref())
830                .await?;
831        if base_branch == branch {
832            bail!("`{branch}` is the base branch; there is nothing to review against");
833        }
834        let fetched_base = resolve_base(
835            &repo,
836            &base_branch,
837            &config.merge.remote,
838            config.coordination.as_ref(),
839        )
840        .await?;
841        let base_commit = fetched_base.commit.clone();
842
843        let roles = config.resolve_roles()?;
844        let max_parallel = config.graph.max_parallel.max(1);
845        let mut state = RunState::new(
846            repo.clone(),
847            base_branch,
848            base_commit.clone(),
849            format!("Review the work already on branch `{branch}`."),
850            config,
851        );
852        state.fetched_base = Some(fetched_base);
853        state.set_origin(origin)?;
854        state.task_instruction = takeover.as_ref().and_then(|t| t.task_instruction.clone());
855        let startup = StartupGuard::reserve(&mut state, "review", Some(branch))?;
856        let opened = async {
857            startup.stage(&mut state, "handover")?;
858
859            // Released before anything else touches the branch: a stale local
860            // branch is moved with `git branch -f`, which git refuses while an
861            // earlier attempt's worktree still has it checked out. Everything
862            // after this point that can fail puts the old run back.
863            // A hand-run review has no task, hence no earlier attempts, but a
864            // worktree of a dead run magi made may still be released.
865            let takeover = takeover.unwrap_or_else(|| crate::handover::Takeover {
866                earlier: Vec::new(),
867                home: crate::run::home(),
868                choice: None,
869                task_instruction: None,
870            });
871            let wait = repository_wait(&state, None);
872            let coordination_config = state.config.coordination.clone();
873            let coordination =
874                crate::repo_lock::SeatLease::operation_with(&repo, coordination_config.as_ref(), &wait)
875                    .await?;
876            let released = crate::handover::release(&repo, branch, &state.id, &takeover).await?;
877            if let Some(released) = &released {
878                state.event(
879                    "release",
880                    format!(
881                        "took `{branch}` over from run {}: its worktree was released: {}",
882                        crate::run::short_of(&released.old_id),
883                        released.audit
884                    ),
885                );
886            }
887            let opened = async {
888                startup.stage(&mut state, "review-preflight")?;
889                // The owner's answer to an earlier divergence question is applied
890                // here: after the release (git will not move a checked-out branch)
891                // and before the sync that would otherwise ask again.
892                state.foreign_checkout = crate::handover::preflight_review(
893                    &repo,
894                    branch,
895                    &takeover.home,
896                    &state.config.merge.remote,
897                )
898                .await?;
899                if state.foreign_checkout.is_some() && takeover.choice.is_some() {
900                    bail!("cannot reconcile a foreign checkout automatically; resolve its branch first");
901                }
902                if let Some(choice) = takeover.choice.as_ref()
903                    && let Err(e) =
904                        crate::reconcile::apply_choice(&repo, &state.config.merge.remote, branch, choice)
905                            .await
906                {
907                    return Err(e.context("applying the owner's answer about the diverged branch"));
908                }
909                // An earlier attempt of this task that published the task head, and
910                // whose work is the branch under review, hands that head over: the
911                // review run updates the same pull request. Without one (a legacy
912                // `magi/<run>/<label>` PR, a hand-named branch) the branch keeps its
913                // own head and its own open pull request is adopted as before.
914                for id in takeover.earlier.iter().rev() {
915                    if let Ok(old) = RunState::load_under(id, &takeover.home)
916                        && let Some(head) = old.pr_head.clone()
917                        && (head == branch || old.winner().is_some_and(|w| w.branch == branch))
918                    {
919                        state.pr_head = Some(head);
920                        break;
921                    }
922                }
923                drop(coordination);
924                startup.stage(&mut state, "open-review")?;
925                Self::open_review(&repo, branch, &mut state, base_commit, &takeover).await
926            }
927            .await;
928            if opened.is_err()
929                && let Some(released) = &released
930            {
931                // Restoration is ownership checked and best effort. Keep the original
932                // startup cause even if reacquiring coordination is no longer possible.
933                match crate::repo_lock::SeatLease::operation_with(
934                    &repo,
935                    coordination_config.as_ref(),
936                    &wait,
937                )
938                .await
939                {
940                    Ok(_coordination) => {
941                        released.restore(&repo, branch).await;
942                        let outcome = RunState::load_under(&released.old_id, &takeover.home)
943                            .map(|old| {
944                                format!(
945                                    "predecessor released_to={:?}; candidate path exists={}",
946                                    old.released_to,
947                                    old.candidates.iter().any(|c| {
948                                        c.branch == branch && c.worktree.exists()
949                                    }),
950                                )
951                            })
952                            .unwrap_or_else(|error| format!("predecessor could not be inspected: {error:#}"));
953                        state.event("startup-restore", format!("ownership-checked restoration attempted: {outcome}; retained successor assets require inspection"));
954                    }
955                    Err(error) => {
956                        tracing::warn!(
957                            "run {}: could not take the repository lease to restore `{branch}`: {error:#}",
958                            state.id
959                        );
960                        state.event(
961                            "startup-restore",
962                            format!("restoration not attempted: {error:#}"),
963                        );
964                    }
965                }
966            }
967            opened
968        }
969        .await;
970        startup.finish(&mut state, opened)?;
971        Ok(Self {
972            state,
973            roles,
974            sem: Arc::new(Semaphore::new(max_parallel)),
975            pause: Pause::new(),
976            interrupt: Pause::new(),
977        })
978    }
979
980    /// Recover exactly the operator-selected public implementation, without touching old sources.
981    pub async fn recover_pr(
982        repo: &Path,
983        task: &str,
984        selection: crate::pr_recovery::Selection,
985        mut config: Config,
986    ) -> Result<Self> {
987        selection.configure(&mut config)?;
988        let assignment = crate::run::task_assignment_for(task)?;
989        selection.check_task_assignment(task, &selection.instruction, assignment)?;
990        let base_branch =
991            git::merge_base_branch(repo, &config.merge.remote, config.merge.base.as_deref())
992                .await?;
993        let fetched_base = resolve_base(
994            repo,
995            &base_branch,
996            &config.merge.remote,
997            config.coordination.as_ref(),
998        )
999        .await?;
1000        let base_commit = fetched_base.commit.clone();
1001        selection
1002            .verify(repo, &config.merge.remote, &base_branch, &selection.head)
1003            .await?;
1004        Box::pin(selection.verify_source(repo)).await?;
1005        if let Some(proof) = &selection.source_proof {
1006            anyhow::ensure!(
1007                git::is_ancestor(repo, &proof.new_base, &base_commit).await,
1008                "source rebase base is outside the fetched landing base"
1009            );
1010        }
1011        let roles = config.resolve_roles()?;
1012        let max_parallel = config.graph.max_parallel.max(1);
1013        let mut state = RunState::new(
1014            repo.to_path_buf(),
1015            base_branch,
1016            base_commit.clone(),
1017            selection.instruction.clone(),
1018            config,
1019        );
1020        state.fetched_base = Some(fetched_base);
1021        state.set_origin(crate::run::Origin::queue(task))?;
1022        state.pr_head = Some(selection.branch.clone());
1023        state.pr_recovery = Some(selection.clone());
1024        Self::open_selected(repo, state, roles, max_parallel, base_commit, selection).await
1025    }
1026
1027    async fn open_selected(
1028        repo: &Path,
1029        mut state: RunState,
1030        roles: ResolvedRoles,
1031        max_parallel: usize,
1032        base_commit: String,
1033        selection: crate::pr_recovery::Selection,
1034    ) -> Result<Self> {
1035        let startup = StartupGuard::reserve(&mut state, "pr-recovery", Some(&selection.branch))?;
1036        let opened = async {
1037            startup.stage(&mut state, "predecessors")?;
1038            crate::pr_head::check_predecessors(&state, &selection.branch).await?;
1039            startup.stage(&mut state, "open-selected")?;
1040            let branch = state.branch_for('A');
1041            let wait = repository_wait(&state, None);
1042            let prepared = async {
1043                let coordination = coordinate_operation(&mut state, repo, &wait).await?;
1044                git::git(repo, &["branch", &branch, selection.source()]).await?;
1045                drop(coordination);
1046                anyhow::Ok(())
1047            }
1048            .await;
1049            prepared?;
1050            state.event(
1051                "recovery",
1052                format!(
1053                    "selected implementation {} for {} (public lease {})",
1054                    selection.source(),
1055                    selection.url,
1056                    selection.head
1057                ),
1058            );
1059            let takeover = crate::handover::Takeover {
1060                earlier: Vec::new(),
1061                home: crate::run::home(),
1062                choice: None,
1063                task_instruction: Some(selection.instruction.clone()),
1064            };
1065            state.save()?;
1066            Self::open_review(repo, &branch, &mut state, base_commit, &takeover).await
1067        }
1068        .await;
1069        startup.finish(&mut state, opened)?;
1070        Ok(Self {
1071            state,
1072            roles,
1073            sem: Arc::new(Semaphore::new(max_parallel)),
1074            pause: Pause::new(),
1075            interrupt: Pause::new(),
1076        })
1077    }
1078
1079    /// What the newest earlier attempt that reviewed `branch` leaves for this
1080    /// run (see [`crate::recovery`]). `None` for a hand-run review, which has
1081    /// no earlier attempts. Reads the earlier record only: ownership of its
1082    /// worktree and branch is [`crate::handover`]'s business.
1083    async fn inherit(
1084        repo: &Path,
1085        branch: &str,
1086        base_commit: &str,
1087        takeover: &crate::handover::Takeover,
1088    ) -> Option<crate::recovery::RecoveryRecord> {
1089        use crate::recovery::{Relation, assess};
1090        let newest = takeover.earlier.last()?;
1091        let mut chosen = None;
1092        for id in takeover.earlier.iter().rev() {
1093            if let Ok(old) = RunState::load_under(id, &takeover.home)
1094                && !old.reviews.is_empty()
1095                && (old.winner().is_some_and(|w| w.branch == branch)
1096                    || old.pr_head.as_deref() == Some(branch))
1097            {
1098                chosen = Some(old);
1099                break;
1100            }
1101        }
1102        let tip = git::rev_parse(repo, &format!("refs/heads/{branch}"))
1103            .await
1104            .unwrap_or_default();
1105        let relation = match chosen.as_ref().and_then(|o| o.reviews.last()) {
1106            Some(last) if last.head == tip => Relation::Same,
1107            Some(last) if git::is_ancestor(repo, &last.head, &tip).await => Relation::Descendant,
1108            _ => Relation::Diverged,
1109        };
1110        let from = chosen.as_ref().map_or(newest.as_str(), |o| o.id.as_str());
1111        Some(assess(from, chosen.as_ref(), base_commit, relation))
1112    }
1113
1114    /// The half of [`Runner::review_taking_over`] that can fail after an
1115    /// earlier attempt's worktree was released.
1116    async fn open_review(
1117        repo: &Path,
1118        branch: &str,
1119        state: &mut RunState,
1120        base_commit: String,
1121        takeover: &crate::handover::Takeover,
1122    ) -> Result<()> {
1123        startup_stage(state, "review-sync")?;
1124        let wait = repository_wait(state, None);
1125        let _coordination = coordinate_operation(state, repo, &wait).await?;
1126        if state.foreign_checkout.is_some() {
1127            // Never reconcile or force-move a ref checked out by the operator.
1128            if state.pr_head.is_none() {
1129                state.pr_head = Some(branch.to_owned());
1130            }
1131        } else if state.pr_recovery.is_none() {
1132            sync_review_branch(repo, branch, &state.config.merge.remote, &base_commit).await?;
1133        }
1134        // The commit subjects are the closest thing to a task statement that
1135        // existing work carries, and the reviewers are told as much.
1136        let start = review_base(
1137            repo,
1138            &state.config.merge.remote,
1139            &state.base_branch,
1140            &base_commit,
1141            branch,
1142        )
1143        .await;
1144        let log = git::log_oneline(repo, &start, branch)
1145            .await
1146            .unwrap_or_default();
1147        let log_text = if log.trim().is_empty() {
1148            "(no commit messages)"
1149        } else {
1150            log.trim()
1151        };
1152        let mut instruction = format!(
1153            "Review the work already on branch `{branch}`. There is no task \
1154             statement: what the change claims to do is whatever its commits \
1155             say.\n\n{log_text}"
1156        );
1157        // A retry of a task whose earlier attempt left reviews behind carries
1158        // that attempt's evidence, and the task's own statement with it: the
1159        // completion criteria are the task's, not whatever the commits claim.
1160        if let Some(mut rec) = Self::inherit(repo, branch, &base_commit, takeover).await {
1161            if let Some(task) = takeover
1162                .task_instruction
1163                .as_deref()
1164                .filter(|t| !t.trim().is_empty())
1165                && rec.reviewed_head.is_some()
1166            {
1167                instruction = format!(
1168                    "{}\n\n# Work already on branch `{branch}` (commit subjects)\n\n{log_text}",
1169                    task.trim()
1170                );
1171                rec.criteria_inherited = true;
1172            }
1173            state.event("recovery", rec.summary());
1174            state.recovery = Some(rec);
1175        }
1176        if let Some(selected) = &state.pr_recovery {
1177            instruction = selected.instruction.clone();
1178        }
1179        state.task_instruction = takeover.task_instruction.clone();
1180        state.instruction = instruction;
1181        state.reviewed_commits = Some(
1182            git::subjects(repo, &start, branch)
1183                .await
1184                .unwrap_or_default(),
1185        );
1186
1187        // A takeover must not forget a hold that a previous review observed,
1188        // even if repairs have since removed it from the branch.
1189        for id in &takeover.earlier {
1190            if let Ok(old) = RunState::load_under(id, &takeover.home)
1191                && old.repo == state.repo
1192                && (old.pr_head.as_deref() == Some(branch)
1193                    || old.winner().is_some_and(|w| w.branch == branch))
1194            {
1195                for hold in old.branch_owner_holds {
1196                    if !state.branch_owner_holds.contains(&hold) {
1197                        state.event(
1198                            "approval-policy",
1199                            format!(
1200                                "inherited {}=true from the reviewed branch's {} at {} (run {id})",
1201                                hold.control.key(),
1202                                hold.file,
1203                                hold.head
1204                            ),
1205                        );
1206                        state.branch_owner_holds.push(hold);
1207                    }
1208                }
1209            }
1210        }
1211        // Select the actual review tip before observing owner policy. A foreign
1212        // checkout can lag its remote descendant, and repairs must not erase
1213        // holds introduced only on that selected commit.
1214        let reviewed_head = if let Some(checkout) = &state.foreign_checkout {
1215            let tracking = format!("{}/{branch}", state.config.merge.remote);
1216            // A surviving tracking ref is not evidence that the remote still
1217            // offers this tip. Adopt it only after a successful fetch here.
1218            let fetched = git::fetch(repo, &state.config.merge.remote, branch).await;
1219            let mut head = checkout.synced_head.clone();
1220            if matches!(fetched, Ok(out) if out.ok()) {
1221                let remote = git::rev_parse(repo, &tracking).await?;
1222                if git::is_ancestor(repo, &checkout.synced_head, &remote).await {
1223                    head = remote;
1224                }
1225            }
1226            head
1227        } else {
1228            git::rev_parse(repo, branch).await?
1229        };
1230        state.observe_branch_owner_holds(&reviewed_head).await;
1231
1232        // An attached worktree, so the fixer's commits land on the branch under
1233        // review rather than on a detached head nobody will look at again.
1234        let worktree = state.worktree_root().join("under-review");
1235        if let Some(parent) = worktree.parent() {
1236            tokio::fs::create_dir_all(parent).await.ok();
1237        }
1238        let path = worktree.to_string_lossy().to_string();
1239        let review_branch = if state.foreign_checkout.is_some() {
1240            let shadow = format!("magi/{}/review", state.short());
1241            git::git(repo, &["branch", &shadow, &reviewed_head]).await?;
1242            shadow
1243        } else {
1244            branch.to_owned()
1245        };
1246        startup_stage(state, "review-checkout")?;
1247        git::git(repo, &["worktree", "add", &path, &review_branch])
1248            .await
1249            .with_context(|| {
1250                format!("checking out `{branch}` at {path} (is it checked out elsewhere?)")
1251            })?;
1252
1253        startup_stage(state, "worktree-setup")?;
1254        let setup = match ensure_setup_config(state, repo).await {
1255            Ok(()) => worktree_setup::prepare(&state.config, repo, &worktree).await,
1256            Err(e) => Err(e),
1257        };
1258        if let Err(e) = setup {
1259            git::worktree_remove_clean(repo, &worktree).await.ok();
1260            return Err(e);
1261        }
1262        let commits = git::commits_ahead(&worktree, &base_commit, "HEAD")
1263            .await
1264            .unwrap_or(0);
1265        if commits == 0 {
1266            git::worktree_remove_clean(repo, &worktree).await.ok();
1267            bail!("`{branch}` has no commits beyond {}", short(&base_commit));
1268        }
1269        let files = git::changed_files(&worktree, &base_commit, "HEAD")
1270            .await
1271            .map(|f| f.len())
1272            .unwrap_or(0);
1273        if files == 0
1274            && let (Ok(head_tree), Ok(base_tree)) = (
1275                git::tree_of(&worktree, "HEAD").await,
1276                git::tree_of(&worktree, &base_commit).await,
1277            )
1278            && head_tree == base_tree
1279        {
1280            let head = git::rev_parse(&worktree, "HEAD").await.unwrap_or_default();
1281            git::worktree_remove_clean(repo, &worktree).await.ok();
1282            bail!(
1283                "`{branch}` at {} has a tree identical to base {}; this usually means \
1284                 the branch ref is stale (check `git rev-parse refs/heads/{branch}` \
1285                 against `{}/{branch}`) rather than an empty change",
1286                short(&head),
1287                short(&base_commit),
1288                state.config.merge.remote
1289            );
1290        }
1291        let stat = git::diff_stat(&worktree, &base_commit, "HEAD")
1292            .await
1293            .unwrap_or_default();
1294
1295        state.candidates.push(Candidate {
1296            index: 0,
1297            label: 'A',
1298            // Not an agent id on purpose: nothing in the roster wrote this, and
1299            // the stats tables must not credit anyone with a win for it.
1300            agent: EXISTING_BRANCH.to_owned(),
1301            branch: review_branch,
1302            worktree,
1303            summary: String::new(),
1304            stat,
1305            files,
1306            commits,
1307            empty: false,
1308            failed: None,
1309            verified_noop: None,
1310            duration_ms: 0,
1311            folded: false,
1312        });
1313        state.tally = Some(Tally {
1314            first_choice: BTreeMap::from([('A', 0)]),
1315            borda: BTreeMap::new(),
1316            winner: 'A',
1317            rankings: 0,
1318            unanimous_initial: false,
1319            deliberated: false,
1320            changed_votes: 0,
1321            unanimous_final: false,
1322            tie_break: None,
1323            // No panel sat, so no quorum applies. Zero judges is the correct
1324            // number for work that never competed, and must not be reported as
1325            // a collapsed panel.
1326            judges: 0,
1327            present: 0,
1328            quorum: 0,
1329            met_quorum: true,
1330            uncontested: Some("review-only run: nothing competed".to_owned()),
1331        });
1332        state.status = RunStatus::Reviewing;
1333        state.prep_complete = Some(true);
1334        state.event(
1335            "start",
1336            format!(
1337                "review-only run {} on `{branch}` ({files} files, {commits} commits)",
1338                state.id
1339            ),
1340        );
1341        Ok(())
1342    }
1343
1344    /// Authorize exactly one additional rebase, under the caller's queue task
1345    /// claim and the same owner/execution locks used by run maintenance.
1346    pub async fn authorize_base_recovery(home: &Path, id: &str, task: &str) -> Result<()> {
1347        Self::authorize_base_recovery_request(home, id, task, false).await
1348    }
1349
1350    /// Queue release uses its durable approval generation exactly once.
1351    pub(crate) async fn authorize_base_recovery_from_resumption(
1352        home: &Path,
1353        id: &str,
1354        task: &str,
1355    ) -> Result<()> {
1356        Self::authorize_base_recovery_request(home, id, task, true).await
1357    }
1358
1359    async fn authorize_base_recovery_request(
1360        home: &Path,
1361        id: &str,
1362        task: &str,
1363        resumption: bool,
1364    ) -> Result<()> {
1365        let _owner = crate::pr_head::owner_lock(home, id)?;
1366        let mut state = RunState::load_under(id, home)?;
1367        crate::pr_head::ensure_owner(&state)?;
1368        let queued = crate::queue::Queue::at(home.join("queue")).get(task)?;
1369        anyhow::ensure!(
1370            queued.execution_runs().last().is_some_and(|run| run == id)
1371                && queued.status != crate::queue::TaskStatus::Done
1372                && !queued.fresh_start
1373                && queued.review_branch.is_none(),
1374            "run task ownership changed"
1375        );
1376        anyhow::ensure!(
1377            state
1378                .origin
1379                .as_ref()
1380                .and_then(|o| o.task.as_deref())
1381                .is_none_or(|owner| owner == task),
1382            "run belongs to another task"
1383        );
1384        let claimed = crate::daemon::is_working_on_other(home, id, Timestamp::now())?;
1385        anyhow::ensure!(
1386            state.liveness_under(home, claimed) == crate::run::Liveness::Dead,
1387            "run is live or its ownership is unknown"
1388        );
1389        // Judge the previous driver's lease before acquiring our maintenance
1390        // lease. The owner lock excludes new driver claims across both steps.
1391        let _execution = crate::driver::maintenance(home, id)?;
1392        anyhow::ensure!(
1393            state.status.resumable()
1394                && state.released_to.is_none()
1395                && state.retired_to.is_none()
1396                && state.pr_taken_over_by.is_none(),
1397            "run ownership is final"
1398        );
1399        anyhow::ensure!(state.base_sync_blocked(), "run has no base-sync stop");
1400        anyhow::ensure!(
1401            !state.pr_publication_pending
1402                && !crate::handover::landing_checkpoint(&state)
1403                && state.merge_decision.is_none(),
1404            "publication or landing has its own retained checkpoint; base recovery cannot replace it"
1405        );
1406        let winner = state.winner().context("run has no retained winner")?;
1407        anyhow::ensure!(
1408            git::is_clean(&winner.worktree).await?,
1409            "retained worktree has uncommitted changes"
1410        );
1411        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1412        anyhow::ensure!(
1413            git::rev_parse(&state.repo, &format!("refs/heads/{}", winner.branch)).await? == head,
1414            "retained worktree does not match the winner branch"
1415        );
1416        state.ensure_base_episode();
1417        state.record_base_sync_stop(&head);
1418        state.save()?;
1419        let fetched =
1420            git::fetch(&state.repo, &state.config.merge.remote, &state.base_branch).await?;
1421        anyhow::ensure!(
1422            fetched.ok(),
1423            "cannot authorize base recovery without a fetched base"
1424        );
1425        let approved_tip = git::rev_parse(
1426            &state.repo,
1427            &format!("{}/{}", state.config.merge.remote, state.base_branch),
1428        )
1429        .await?;
1430        if resumption {
1431            anyhow::ensure!(
1432                crate::daemon::compatible_run_instruction(&state, &queued),
1433                "resumption instruction does not match the retained run"
1434            );
1435            let approval = queued
1436                .resumption
1437                .as_ref()
1438                .context("no operator resumption")?;
1439            anyhow::ensure!(
1440                approval.instruction == queued.instruction,
1441                "resumption approval does not match the current task instruction"
1442            );
1443            state.authorize_base_sync_resumption(task, approval)?;
1444        } else {
1445            state.authorize_base_sync(task)?;
1446        }
1447        let episode = state.base_sync_recovery.episodes.last().unwrap().id;
1448        state
1449            .base_sync_recovery
1450            .authorizations
1451            .last_mut()
1452            .unwrap()
1453            .scope = Some(crate::base_sync_recovery::Scope {
1454            tip: approved_tip,
1455            instruction: queued.instruction.clone(),
1456            episode,
1457        });
1458        state.event(
1459            "base-recovery",
1460            "owner authorized a bounded base synchronization interval; fresh verification required",
1461        );
1462        state.save()?;
1463        Ok(())
1464    }
1465
1466    /// Reopen an existing run.
1467    pub fn resume(id: &str) -> Result<Self> {
1468        let state = RunState::load(id)?;
1469        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &state.id)?;
1470        if state.startup.as_ref().is_some_and(|s| !s.complete)
1471            || (state.status == RunStatus::Prep
1472                && state.instruction.trim().is_empty()
1473                && state.candidates.is_empty()
1474                && state.prep_complete == Some(false))
1475        {
1476            bail!(
1477                "run {id} did not finish startup; inspect its startup evidence and retained assets, then start an explicitly authorized review or recovery"
1478            );
1479        }
1480        crate::pr_head::ensure_owner(&state)?;
1481        if state.pr_recovery.is_some() && state.reviewed_commits.is_none() {
1482            bail!(
1483                "selected public-head recovery did not finish setup; use `magi task recover-pr` after correcting the cause"
1484            );
1485        }
1486        if let Some(to) = &state.released_to {
1487            bail!(
1488                "run {} cannot be resumed: its worktree was released to run {}",
1489                state.short(),
1490                crate::run::short_of(to)
1491            );
1492        }
1493        let roles = state.config.resolve_roles()?;
1494        if let Some(id) = &state.config.graph.selected_implementer
1495            && state.candidates.iter().any(|c| c.agent != *id)
1496        {
1497            bail!(
1498                "replacement implementer pin does not match the saved candidates; task must stay held"
1499            );
1500        }
1501        let max_parallel = state.config.graph.max_parallel.max(1);
1502        Ok(Self {
1503            state,
1504            roles,
1505            sem: Arc::new(Semaphore::new(max_parallel)),
1506            pause: Pause::new(),
1507            interrupt: Pause::new(),
1508        })
1509    }
1510
1511    /// Walk the graph to a terminal state, skipping nodes already recorded.
1512    ///
1513    /// Every way a run is driven - the queue loop, `magi run`, a resume from
1514    /// the phone - ends here, so this is the one place a run that ended
1515    /// Blocked / Stalled / Failed, or died with an error, is announced to the
1516    /// attention-report outbox. Best-effort: see [`crate::notices::raise`].
1517    pub async fn execute(&mut self) -> Result<()> {
1518        anyhow::ensure!(
1519            self.state.startup.as_ref().is_none_or(|s| s.complete),
1520            "startup incomplete; inspect retained evidence before explicit recovery"
1521        );
1522        if let Some(selection) = &self.state.pr_recovery
1523            && (selection.source_proof.is_some() || selection.source_run.is_some())
1524        {
1525            Box::pin(selection.verify_source(&self.state.repo)).await?;
1526        }
1527        // Queue claims cover daemon work; this also serializes manual drivers
1528        // consuming a repair answer, including the new review and decision.
1529        let _repair_claim = if self.state.config.graph.selected_implementer.is_some()
1530            || self.state.land_approval.is_some()
1531            || self.state.merge_decision.is_some()
1532            || !self.state.merge_repairs.is_empty()
1533            || (self.state.status == RunStatus::Landing && land::repair_available(&self.state))
1534        {
1535            let claim = FixClaim::acquire(&self.state.dir())?;
1536            crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
1537            let disk = RunState::load(&self.state.id)?;
1538            let checkpoint = |s: &RunState| {
1539                (
1540                    s.merge_repairs.len(),
1541                    s.merge_repairs
1542                        .last()
1543                        .map(|r| (r.question.clone(), r.phase)),
1544                    s.land_approval
1545                        .as_ref()
1546                        .map(|a| (a.question.clone(), a.head.clone())),
1547                )
1548            };
1549            if checkpoint(&disk) != checkpoint(&self.state) {
1550                bail!(
1551                    "repair decision changed since this run was loaded; reload and resume the existing run, never start a second repair"
1552                );
1553            }
1554            Some(claim)
1555        } else {
1556            None
1557        };
1558        let mut execution = self.claim_driver().await?;
1559        let runner = &mut *execution.runner;
1560        let wait = runner.repository_wait();
1561        // Keep the graph's large future off the callers' stack, including
1562        // standalone task attempts running on a default-sized test thread.
1563        let result = wait.scope(Box::pin(runner.execute_graph())).await;
1564        let result = match result {
1565            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
1566                record_repository_block(&mut runner.state, &e);
1567                runner.park_here()?;
1568                Ok(())
1569            }
1570            other => other,
1571        };
1572        execution.driver.finish(&mut runner.state).await?;
1573        let ended = if result.is_err() {
1574            Some({
1575                let mut notice = crate::notices::run_stopped(&runner.state.id, &runner.state);
1576                if let Err(error) = &result {
1577                    notice.message = format!(
1578                        "Run {} stopped with an error: {}",
1579                        runner.state.id,
1580                        error.to_string().lines().next().unwrap_or("unknown error")
1581                    );
1582                }
1583                notice
1584            })
1585        } else {
1586            crate::notices::run_ended(&runner.state)
1587        };
1588        if let Some(notice) = ended {
1589            crate::notices::raise(notice);
1590        }
1591        result
1592    }
1593
1594    fn repository_wait(&self) -> crate::repo_lock::WaitPolicy {
1595        repository_wait(&self.state, Some((&self.pause, &self.interrupt)))
1596    }
1597
1598    async fn coordinate(&mut self, cwd: &Path) -> Result<Option<crate::repo_lock::SeatLease>> {
1599        let wait = self.repository_wait();
1600        coordinate_operation(&mut self.state, cwd, &wait).await
1601    }
1602
1603    /// Validate ownership and persist the driver claim in one critical section.
1604    async fn claim_driver(&mut self) -> Result<DriverExecution<'_>> {
1605        let owner = match crate::pr_head::owner_lock_wait(
1606            &crate::run::home(),
1607            &self.state.id,
1608            Duration::from_secs(self.state.config.graph.timeout_resource_wait),
1609        )
1610        .await
1611        {
1612            Ok(owner) => owner,
1613            Err(error) => {
1614                let detail = format!("worktree maintenance or ownership update: {error:#}");
1615                // Admission did not own run.json: retain this resource outcome
1616                // in memory for queue settlement without overwriting a concurrent
1617                // retirement or publication transaction.
1618                self.state.resource_block = Some(crate::run::ResourceBlock {
1619                    cause: crate::run::ResourceBlockCause::Other,
1620                    node: "start".into(),
1621                    kind: "worktree maintenance".into(),
1622                    detail: detail.clone(),
1623                    at: Timestamp::now(),
1624                });
1625                return Err(crate::repo_lock::ResourceBlocked(detail).into());
1626            }
1627        };
1628        let disk = RunState::load(&self.state.id)?;
1629        crate::pr_head::ensure_owner(&disk)?;
1630        let claimed =
1631            crate::daemon::is_working_on_other(&crate::run::home(), &disk.id, Timestamp::now())?;
1632        // Only proven-live ownership refuses here. Unknown records (legacy runs
1633        // with no claim marker or process identity) fall through to the
1634        // exclusive execution lease taken just below, which is what actually
1635        // keeps a second driver out.
1636        anyhow::ensure!(
1637            disk.liveness(claimed) != crate::run::Liveness::Live,
1638            "run already has a live driver; wait for its execution to end"
1639        );
1640        if let Some(to) = &disk.released_to {
1641            bail!(
1642                "run {} cannot continue: its worktree was released to run {}",
1643                self.state.short(),
1644                crate::run::short_of(to)
1645            );
1646        }
1647        anyhow::ensure!(
1648            serde_json::to_value(&disk.base_sync_recovery)?
1649                == serde_json::to_value(&self.state.base_sync_recovery)?,
1650            "base recovery changed since this run was loaded; reload the saved run"
1651        );
1652        let driver = crate::driver::DriverGuard::acquire(&crate::run::home(), &disk)?;
1653        let mut execution = DriverExecution {
1654            runner: self,
1655            driver,
1656        };
1657        // Declare after the execution guard: on any claim error this short
1658        // lock must drop before that guard reloads and releases ownership.
1659        let _owner = owner;
1660        crate::repair_budget::apply(&crate::run::home(), &mut execution.runner.state, &disk)
1661            .await?;
1662        let state = &mut execution.runner.state;
1663        // Moving again, so it is no longer parked. Set before the walk rather
1664        // than in `resume`, so every way of re-entering the graph clears it
1665        // and a card cannot claim a run is waiting to be resumed while the
1666        // agents are already working.
1667        state.parked = false;
1668        state.resource_wait = None;
1669        state.resource_block = None;
1670        // Any seat this state still lists as answering belongs to whatever
1671        // process last drove this run — this one included, if it crashed
1672        // mid-wave. Cleared and flushed immediately, before anything else
1673        // runs, so a resume can never show a seat as live when nothing is
1674        // asking it anything yet; the node that actually dispatches the next
1675        // wave repopulates it.
1676        state.clear_active();
1677        // Recorded in the same spot, and flushed together with the clear
1678        // above: this is the pid a reader checks (`RunState::liveness`) when
1679        // no daemon claim exists to answer "is a process still driving this
1680        // run" — a plain `magi run` / `magi review` typed into a terminal
1681        // claims nothing there. Always overwritten, never only-if-absent, so
1682        // a resumed run's stale pid from a previous, possibly-dead process
1683        // can never survive into this one's own report. Unlike
1684        // `clear_active`, this changes on every single `execute()` call, so
1685        // the save below is now unconditional rather than only-if-cleared.
1686        //
1687        // `driver_started_at` is recorded in the same breath, from this same
1688        // pid, so `liveness` can tell a live pid that is genuinely still us
1689        // apart from one the OS has since handed to an unrelated process —
1690        // see that field's own doc for why the pid alone is not enough.
1691        let pid = std::process::id();
1692        state.driver_pid = Some(pid);
1693        state.driver_started_at = crate::proc::process_started_at(pid);
1694        state.driver_exited = false;
1695        state.driver_claimed = Some(true);
1696        state.begin_pass();
1697        execution.driver.bind(state);
1698        state.save()?;
1699        execution.driver.record(state)?;
1700        Ok(execution)
1701    }
1702
1703    /// No conflict checkpoint proves dispatch was never reached. End that
1704    /// consumed generation so an owner can explicitly authorize a fresh one.
1705    async fn repair_base_conflict(&mut self, dispatch: bool) -> Result<()> {
1706        let result = crate::base_conflict::repair(&mut self.state, dispatch).await;
1707        if let Err(error) = &result
1708            && !error.is::<crate::repo_lock::ResourceBlocked>()
1709            && self
1710                .state
1711                .repair_grants
1712                .last()
1713                .is_some_and(|g| g.conflict.is_none())
1714        {
1715            self.state.repair_grants.last_mut().unwrap().phase =
1716                crate::repair_budget::Phase::Complete;
1717            self.state.status = RunStatus::Blocked;
1718            self.state.save()?;
1719            bail!(
1720                "{error:#}. The consumed reservation ended before editing-seat dispatch. Inspect its scratch tree, then authorize a new base-conflict grant with a new request ID and current --expect-pass {}; the old request will not dispatch again",
1721                self.state.passes.last().map_or(0, |p| p.index)
1722            );
1723        }
1724        result
1725    }
1726
1727    /// A grant owns only these repair attempts. Confirmation reviews cannot
1728    /// create another fixer allowance, and every operation is restart-checkpointed.
1729    async fn execute_granted_repair(&mut self) -> Result<()> {
1730        use crate::repair_budget::{Kind, Phase};
1731        let index = self.state.repair_grants.len() - 1;
1732        let winner = self
1733            .state
1734            .winner()
1735            .cloned()
1736            .context("repair winner missing")?;
1737        loop {
1738            let grant = self.state.repair_grants[index].clone();
1739            match grant.phase {
1740                Phase::Pending => {
1741                    if grant.spent >= grant.receipt.request.rounds {
1742                        self.state.repair_grants[index].phase = Phase::Complete;
1743                        self.state.status = RunStatus::Blocked;
1744                        self.state.save()?;
1745                        return Ok(());
1746                    }
1747                    // Reserve before dispatch. A crash can consume an attempt,
1748                    // but can never duplicate it or reset saved fixer history.
1749                    self.state.repair_grants[index].spent += 1;
1750                    self.state.repair_grants[index].phase = Phase::Fixing;
1751                    self.state.save()?;
1752                    match grant.receipt.request.kind {
1753                        Kind::BaseConflict => self.repair_base_conflict(true).await?,
1754                        Kind::Gate => {
1755                            let failed = self.state.gate.clone();
1756                            let _ = self.gate_fix_round(&winner, &failed).await?;
1757                        }
1758                        Kind::Review => {
1759                            let mut record = self
1760                                .state
1761                                .reviews
1762                                .last()
1763                                .cloned()
1764                                .context("review failure missing")?;
1765                            let round = record.round;
1766                            let step = self.run_fix_round(&winner, &mut record).await?;
1767                            self.state.reviews[round - 1] = record;
1768                            if matches!(step, FixStep::Recorded { report_lost: true }) {
1769                                self.state.repair_grants[index].phase = Phase::Complete;
1770                                self.state.status = RunStatus::Blocked;
1771                                self.state.event("repair-grant", "fixer continuation remains unresolved; repair grant cannot bypass its stop");
1772                                self.state.save()?;
1773                                return Ok(());
1774                            }
1775                            if matches!(step, FixStep::Resource) {
1776                                self.stop_for_resource()?;
1777                                return Ok(());
1778                            }
1779                            self.state.save()?;
1780                        }
1781                    }
1782                    if self.resource_halt()? {
1783                        return Ok(());
1784                    }
1785                }
1786                Phase::Fixing => {
1787                    if grant.receipt.request.kind == Kind::BaseConflict {
1788                        self.repair_base_conflict(false).await?;
1789                    }
1790                    // Interrupted fixers may have left source changes or an
1791                    // unpushed commit. Preserve them before fresh assessment.
1792                    let _coordination = self.coordinate(&winner.worktree).await?;
1793                    let rescued = git::rescue_commit(
1794                        &winner.worktree,
1795                        "magi: preserve interrupted granted repair",
1796                    )
1797                    .await?;
1798                    self.state.note_withheld("repair-grant", &rescued.withheld);
1799                    // Fresh checks must supersede even a same-head failure.
1800                    // Archive rather than erase failure evidence or counters.
1801                    self.state
1802                        .verification_history
1803                        .push(crate::run::VerificationEvidence {
1804                            superseded_by: None,
1805                            archived_at: Some(Timestamp::now()),
1806                            fixes_spent: Some(self.state.gate_fixes.len()),
1807                            gate_head: self.state.gate_head.take(),
1808                            gate_ran: std::mem::take(&mut self.state.gate_ran),
1809                            gate: std::mem::take(&mut self.state.gate),
1810                            e2e: self.state.head_e2e.take(),
1811                        });
1812                    if let Some(e2e) = self.state.gate_repair_e2e.take() {
1813                        self.state
1814                            .verification_history
1815                            .push(crate::run::VerificationEvidence {
1816                                superseded_by: None,
1817                                archived_at: Some(Timestamp::now()),
1818                                fixes_spent: Some(self.state.gate_fixes.len()),
1819                                gate_head: None,
1820                                gate_ran: false,
1821                                gate: Vec::new(),
1822                                e2e: Some(e2e),
1823                            });
1824                    }
1825                    self.state.repair_grants[index].review_start = self.state.reviews.len();
1826                    self.state.repair_grants[index].head =
1827                        git::rev_parse(&winner.worktree, "HEAD").await?;
1828                    self.state.repair_grants[index].phase = Phase::Reviewing;
1829                    if grant.receipt.request.kind == Kind::BaseConflict {
1830                        self.state.base_sync_recovery.stop = None;
1831                        if let Some(sync) = &mut self.state.base_sync {
1832                            sync.tip = grant
1833                                .receipt
1834                                .request
1835                                .tip
1836                                .clone()
1837                                .context("repair base tip missing")?;
1838                            sync.conflict = None;
1839                            sync.behind = 0;
1840                        }
1841                        refresh_reviewed_commits(&mut self.state, &winner.branch).await;
1842                    }
1843                    self.state.save()?;
1844                }
1845                Phase::Reviewing => {
1846                    self.review_loop_pass().await?;
1847                    if self.resource_halt()? {
1848                        return Ok(());
1849                    }
1850                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1851                    let clean = self.state.reviews.len() > grant.review_start
1852                        && self.state.reviews.last().is_some_and(|r| {
1853                            r.head == head
1854                                && r.clean
1855                                && r.expected > 0
1856                                && r.answered == r.expected
1857                                && matches!(
1858                                    r.e2e_status(),
1859                                    E2eStatus::Passed | E2eStatus::NotConfigured
1860                                )
1861                        });
1862                    if !clean {
1863                        // A confirmation review carries no additional fix budget.
1864                        self.state.repair_grants[index].phase = if grant.receipt.request.kind
1865                            == Kind::Review
1866                            && grant.spent < grant.receipt.request.rounds
1867                            && self.state.reviews.last().is_some_and(|r| {
1868                                r.fix.is_none()
1869                                    && (r.blocking > 0 || r.e2e_status() == E2eStatus::Failed)
1870                            }) {
1871                            Phase::Pending
1872                        } else {
1873                            Phase::Complete
1874                        };
1875                        self.state.status = RunStatus::Blocked;
1876                        self.state.event(
1877                            "repair-grant",
1878                            "fresh review did not clear the repaired head; no publication",
1879                        );
1880                        self.state.save()?;
1881                        if self.state.repair_grants[index].phase == Phase::Pending {
1882                            continue;
1883                        }
1884                        return Ok(());
1885                    }
1886                    self.state.repair_grants[index].phase = Phase::Gating;
1887                    self.state.save()?;
1888                }
1889                Phase::Gating => {
1890                    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
1891                    self.state.reconcile_verification_head(&head);
1892                    let wait = self.repository_wait();
1893                    if self
1894                        .state
1895                        .reviews
1896                        .last()
1897                        .is_none_or(|r| r.head != head || !r.clean)
1898                    {
1899                        self.state.status = RunStatus::Blocked;
1900                        self.state.repair_grants[index].phase = Phase::Complete;
1901                        self.state.event(
1902                            "repair-grant",
1903                            "head changed since fresh review; repair held",
1904                        );
1905                        self.state.save()?;
1906                        return Ok(());
1907                    }
1908                    if !verify_current_head(&mut self.state, &winner, &wait).await? {
1909                        return Ok(());
1910                    }
1911                    let outcomes = self.run_gate(&winner).await?;
1912                    if verify_inconclusive(&outcomes) {
1913                        return Ok(());
1914                    }
1915                    let after = git::rev_parse(&winner.worktree, "HEAD").await?;
1916                    if after != head {
1917                        self.state.repair_grants[index].review_start = self.state.reviews.len();
1918                        self.state.repair_grants[index].head = after;
1919                        self.state.repair_grants[index].phase = Phase::Reviewing;
1920                        self.state.status = RunStatus::Blocked;
1921                        self.state.event("repair-grant", "pre-gate changed the reviewed head; resume for fresh review before publication");
1922                        self.state.save()?;
1923                        return Ok(());
1924                    }
1925                    let passed = outcomes.iter().all(CommandOutcome::ok);
1926                    // Follow-up grants owe the same guarded checkout delivery
1927                    // as the conflict repair that introduced the new head.
1928                    if passed
1929                        && !crate::foreign_checkout::write_back(&mut self.state, &head).await?
1930                    {
1931                        return Ok(());
1932                    }
1933                    self.state.repair_grants[index].succeeded = passed;
1934                    self.state.gate = outcomes;
1935                    self.state.gate_ran = true;
1936                    self.state.status = if passed {
1937                        RunStatus::Gating
1938                    } else {
1939                        RunStatus::Blocked
1940                    };
1941                    self.state.repair_grants[index].phase = if !passed
1942                        && grant.receipt.request.kind == Kind::Gate
1943                        && grant.spent < grant.receipt.request.rounds
1944                    {
1945                        Phase::Pending
1946                    } else {
1947                        Phase::Complete
1948                    };
1949                    if passed && self.state.config.merge.mode == MergeMode::Pr {
1950                        // Persist publication debt with the successful checkpoint.
1951                        self.state.pr_publication_pending = true;
1952                    } else if passed {
1953                        self.state.status = RunStatus::Ready;
1954                    }
1955                    self.state.save()?;
1956                    if !passed {
1957                        continue;
1958                    }
1959                    // Existing PR ownership and push lease stay in place. The
1960                    // normal publisher owes the new reviewed/gated SHA, and
1961                    // land's fresh owner approval remains mandatory.
1962                    if self.state.config.merge.mode != MergeMode::Pr {
1963                        self.state.status = RunStatus::Ready;
1964                        self.state.event(
1965                            "repair-grant",
1966                            "repair verified locally; grant carries no merge authorization",
1967                        );
1968                        self.state.save()?;
1969                        return Ok(());
1970                    }
1971                    return self.merge().await;
1972                }
1973                Phase::Complete => return Ok(()),
1974            }
1975        }
1976    }
1977
1978    async fn execute_graph(&mut self) -> Result<()> {
1979        // Even a terminal run may reenter through an explicit recovery action.
1980        // Restore before prep shortcuts, saved conversations or graph nodes use
1981        // a disposable path; GC may already have removed its original objects.
1982        let archive_home = crate::run::home().canonicalize()?;
1983        if crate::retention::has_archived_seats(&archive_home, &self.state)? {
1984            let repo = self.state.repo.clone();
1985            let _coordination = self.coordinate(&repo).await?;
1986            ensure_setup_config(&mut self.state, &repo).await?;
1987            crate::retention::restore_execution_seats(&archive_home, &self.state).await?;
1988        }
1989        self.state.configure_recovery_resume()?;
1990        if self
1991            .state
1992            .repair_grants
1993            .last()
1994            .is_some_and(|g| g.phase != crate::repair_budget::Phase::Complete)
1995        {
1996            return self.execute_granted_repair().await;
1997        }
1998        if self
1999            .state
2000            .repair_grants
2001            .last()
2002            .is_some_and(|g| !g.succeeded)
2003        {
2004            self.state.status = RunStatus::Blocked;
2005            self.state.save()?;
2006            return Ok(());
2007        }
2008        // Preservation may invalidate an old empty classification. Resolve it
2009        // before any resume shortcut can reuse a verdict or publish a branch.
2010        self.persist_candidates().await?;
2011        // A saved base stop outranks stage markers, including legacy Voting.
2012        // Recovery never reenters implementation, deliberation or voting.
2013        if self.state.base_sync_blocked() {
2014            self.sync_to_base().await?;
2015            if !self.state.base_sync_blocked()
2016                && self.state.status != RunStatus::AlreadyInBase
2017                && !self.resource_halt()?
2018            {
2019                self.finish_after_tally().await?;
2020            }
2021            return Ok(());
2022        }
2023        if self
2024            .state
2025            .merge_decision
2026            .as_ref()
2027            .is_some_and(|decision| !decision.closed)
2028            && self.state.status != RunStatus::Landing
2029        {
2030            if crate::merge_decision::resumable_hold(&self.state) {
2031                self.state.merge = Some(MergeOutcome {
2032                    mode: MergeMode::Pr,
2033                    ok: false,
2034                    detail: self
2035                        .state
2036                        .merge_decision
2037                        .as_ref()
2038                        .unwrap()
2039                        .target
2040                        .pr
2041                        .clone(),
2042                    empty: false,
2043                });
2044                self.run_land().await?;
2045                self.settle_questions();
2046                return Ok(());
2047            }
2048            bail!(
2049                "owner stop decision is held; inspect the existing PR and request an explicit fresh review/decision before further work"
2050            );
2051        }
2052        if self.state.config.graph.selected_implementer.is_some()
2053            && self.state.status == RunStatus::Blocked
2054            && self.state.tally.is_none()
2055            && self.state.candidate_preservation.is_empty()
2056            && self.state.interrupted_implementations.is_empty()
2057        {
2058            bail!(
2059                "selected implementation is held; explicitly release the owning task to retry it"
2060            );
2061        }
2062
2063        if self.state.merge_repairs.last().is_some_and(|r| {
2064            matches!(
2065                r.phase,
2066                crate::run::MergeRepairPhase::Requested
2067                    | crate::run::MergeRepairPhase::Fixing
2068                    | crate::run::MergeRepairPhase::Reviewing
2069            )
2070        }) {
2071            if self.repair_merge_review().await? {
2072                self.run_land().await?;
2073            }
2074            self.settle_questions();
2075            return Ok(());
2076        }
2077        // Publication already passed review and gate. Resume only that node:
2078        // syncing to a newer base here could change the verified commit.
2079        if self.state.pr_publication_pending {
2080            self.merge().await?;
2081            if self.resource_halt()? {
2082                return Ok(());
2083            }
2084            self.state.save()?;
2085            self.settle_questions();
2086            return Ok(());
2087        }
2088        // A run that already lost its quorum never resumes into the verdict
2089        // machinery: `deliberate` and `vote` would otherwise clobber the
2090        // stalled marker back to Voting and the run would keep going past a
2091        // verdict that is no longer trustworthy. Everything already recorded is
2092        // kept, so the run stays resumable (or foldable) for a human to pick up.
2093        //
2094        // On --resume the run gets one chance to repair itself: the seats a
2095        // rate limit took out are re-asked. If their quota has since reset and
2096        // the quorum is restored, the run picks up and finishes; otherwise it
2097        // stays stale and still-resumable for a later retry. If it does not
2098        // recover, the returned status stays `Stalled` and nothing was
2099        // clobbered (the recovery only mutates entries for the lost seats).
2100        if self.state.status == RunStatus::Stalled {
2101            if self.recover_stall().await? {
2102                self.finish_after_tally().await?;
2103            } else {
2104                // Still below quorum: persist the marker and stay resumable.
2105                self.state.save()?;
2106            }
2107            return Ok(());
2108        }
2109        // A run parked inside `land` - watching CI, mid fix-round, or
2110        // waiting on the owner's merge approval - resumes directly into it,
2111        // never back through `prep`. Everything before `merge` already
2112        // concluded; that is the only way `status` reaches `Landing` in the
2113        // first place. Re-walking `review_loop` first would also be actively
2114        // wrong: its own status recomputation (see its doc) treats any
2115        // clean round as reason to set `status` to `Gating`, which would
2116        // clobber this marker before `merge` ever ran, and this run would
2117        // never find its way back into `land` at all.
2118        if self.state.status == RunStatus::Landing
2119            || crate::handover::landing_checkpoint(&self.state)
2120        {
2121            self.state.land_sync_pending = true;
2122            self.state.save()?;
2123            self.run_land().await?;
2124            // `run_land` may have settled the run right here - CI came back
2125            // green and the PR merged, say - without ever passing back
2126            // through `merge`'s own trailing call. Whatever it left `status`
2127            // as is what this has to read.
2128            self.settle_questions();
2129            return Ok(());
2130        }
2131        // A run parked at `merge` on the owner's word about a withheld PR text
2132        // resumes straight into `merge`. Re-walking the tail would sync to the
2133        // base again (possibly rebasing, possibly spending the fixer) after the
2134        // gate had passed, and then push a tree nobody reviewed or gated.
2135        if self.state.github_text.is_some()
2136            && self.state.merge.is_none()
2137            && self.state.status == RunStatus::Gating
2138        {
2139            self.merge().await?;
2140            if self.resource_halt()? {
2141                return Ok(());
2142            }
2143            self.state.save()?;
2144            return Ok(());
2145        }
2146        if let Err(e) = self.prep().await {
2147            if e.is::<crate::repo_lock::ResourceBlocked>() {
2148                return Err(e);
2149            }
2150            self.state.status = RunStatus::Blocked;
2151            self.state
2152                .prep_failure
2153                .get_or_insert_with(|| format!("{e:#}"));
2154            self.state
2155                .event("prep", format!("preparation stopped: {e:#}"));
2156            self.state.save()?;
2157            return Err(e);
2158        }
2159        if self.park_here()? {
2160            return Ok(());
2161        }
2162        self.advise().await?;
2163        if self.park_here()? {
2164            return Ok(());
2165        }
2166        self.implement().await?;
2167        if self.park_here()? {
2168            return Ok(());
2169        }
2170        // `after_implement` already saved the state and settled any open
2171        // questions when it set this; nothing later in the graph has
2172        // anything to judge.
2173        if self.state.status == RunStatus::VerifiedNoop {
2174            return Ok(());
2175        }
2176        self.judge().await?;
2177        if self.park_here()? {
2178            return Ok(());
2179        }
2180        self.deliberate().await?;
2181        if self.park_here()? {
2182            return Ok(());
2183        }
2184        self.vote().await?;
2185        if self.park_here()? {
2186            return Ok(());
2187        }
2188        self.tally()?;
2189        // A verdict that lost its quorum is not trustworthy: do not review,
2190        // gate, or merge on it. Everything already done is kept, so the run
2191        // stays resumable (or foldable); the human can replace the agent that
2192        // ran out of quota and pick it up.
2193        if self.state.status == RunStatus::Stalled {
2194            // Persist the stalled marker now — the normal end-of-execute save
2195            // below is below this early return, and without it a resumed run
2196            // would reload a pre-tally status and keep going.
2197            self.state.save()?;
2198            return Ok(());
2199        }
2200        self.finish_after_tally().await?;
2201        Ok(())
2202    }
2203
2204    /// A wave could not start its agents because a shared resource stayed busy
2205    /// (see [`RunState::resource_block`]): leave the run `Blocked` and
2206    /// resumable. Unstarted seats have no result; completed peers retain
2207    /// their saved candidate state.
2208    fn stop_for_resource(&mut self) -> Result<()> {
2209        // Resource acquisition observes cancellation inside a node. Complete
2210        // the same durable park as every other node boundary before returning.
2211        if self.pause.parked() || self.interrupt.parked() {
2212            self.park_here()?;
2213            return Ok(());
2214        }
2215        if self.state.resource_block.is_some() {
2216            self.state.status = RunStatus::Blocked;
2217        }
2218        self.state.save()
2219    }
2220
2221    /// Stop walking the graph if a node ended on a resource block.
2222    fn resource_halt(&mut self) -> Result<bool> {
2223        if self.state.resource_block.is_none() {
2224            return Ok(false);
2225        }
2226        self.stop_for_resource()?;
2227        Ok(true)
2228    }
2229
2230    /// Park here if asked to, recording it in the run's own timeline.
2231    ///
2232    /// Returns whether the caller should stop walking the graph. The state is
2233    /// saved either way by the node that just finished; this adds the event so
2234    /// the operator's card says why a run that is neither finished nor moving
2235    /// is sitting where it is.
2236    fn park_here(&mut self) -> Result<bool> {
2237        let parking = self.pause.parked() || self.interrupt.parked();
2238        if !parking && self.state.resource_block.is_some() {
2239            self.stop_for_resource()?;
2240            return Ok(true);
2241        }
2242        // Either handle asking is enough - see `Pause`'s own doc for why
2243        // they are never the same one. `interrupt` is checked second so a
2244        // reason it carries is preferred in the message below over a plain
2245        // shutdown park racing it at the same boundary.
2246        if !self.pause.parked() && !self.interrupt.parked() {
2247            return Ok(false);
2248        }
2249        if self.state.parked {
2250            return Ok(true);
2251        }
2252        let cause = self.interrupt.cause().or_else(|| self.pause.cause());
2253        if let Some(block) = &mut self.state.resource_block
2254            && matches!(
2255                block.cause,
2256                crate::run::ResourceBlockCause::Cancelled { .. }
2257            )
2258        {
2259            block.cause = crate::run::ResourceBlockCause::Cancelled {
2260                park: cause.clone(),
2261            };
2262        }
2263        let why = match cause.as_ref().map(ParkCause::label) {
2264            Some(reason) => format!(
2265                "parked after `{}` ({reason}) — resume to carry on from here",
2266                self.state.status.as_str()
2267            ),
2268            None => format!(
2269                "parked after `{}` — resume to carry on from here",
2270                self.state.status.as_str()
2271            ),
2272        };
2273        self.state.event("park", why);
2274        self.state
2275            .end_pass_parked(cause.unwrap_or(ParkCause::Unspecified));
2276        self.state.parked = true;
2277        self.state.save()?;
2278        Ok(true)
2279    }
2280
2281    /// Only a new explicit operator approval may retry failed preparation.
2282    pub fn retry_preparation(&mut self, approval_at: jiff::Timestamp) -> Result<()> {
2283        if self.state.prep_failure.is_none() {
2284            return Ok(());
2285        }
2286        let failed_at = self
2287            .state
2288            .events
2289            .iter()
2290            .rev()
2291            .find(|event| event.node == "prep")
2292            .map_or(self.state.updated_at, |event| event.at);
2293        anyhow::ensure!(
2294            approval_at > failed_at,
2295            "preparation retry requires a new explicit release after the failure"
2296        );
2297        self.state.prep_failure = None;
2298        self.state.prep_complete = Some(false);
2299        self.state
2300            .event("prep-retry", "operator approved retry of preparation");
2301        Ok(())
2302    }
2303
2304    /// Hand the runner the pause `magi serve`'s own shutdown watches.
2305    pub fn on_pause(&mut self, pause: Pause) {
2306        self.pause = pause;
2307    }
2308
2309    /// Hand the runner a second, independent pause: `magi serve`'s interrupt
2310    /// scheduler asking this one run - and no other - to park so a task
2311    /// marked [`crate::queue::Task::interrupt`] can run alone. See
2312    /// [`Pause`]'s own doc for why this is never [`Runner::on_pause`]'s
2313    /// handle.
2314    pub fn watch_interrupt(&mut self, pause: Pause) {
2315        self.interrupt = pause;
2316    }
2317
2318    /// Abandon this run's own open questions, once `status` has actually
2319    /// settled rather than merely paused.
2320    ///
2321    /// `Blocked` and `Stalled` are `RunStatus::resumable` — a human can pick
2322    /// either back up with the candidates, the review round and the seat
2323    /// sessions already on disk, so a question an implementer asked mid-round
2324    /// may still get a real answer read by a real resume. Only the statuses
2325    /// `resumable` excludes are actually final: the run merged, it reached
2326    /// `Ready` with nothing left to do, it failed outright with no
2327    /// established point to continue from, or every candidate agreed, with
2328    /// evidence, that nothing belonged in the worktree (`VerifiedNoop`). In
2329    /// every one of those the seat that asked is gone for good, exactly like
2330    /// the run being deleted under `magi run rm` - so the same cleanup
2331    /// applies, worded for what actually happened instead of "the run was
2332    /// deleted".
2333    ///
2334    /// Best-effort and silent on success: called from every place `status`
2335    /// can land on one of those three, including ones a resumed run revisits,
2336    /// so it must cost nothing when there was nothing open to begin with.
2337    fn settle_questions(&mut self) {
2338        if let Err(e) = ask::Questions::open().settle_run(&self.state.id, self.state.status) {
2339            tracing::warn!("abandon questions for {}: {e:#}", self.state.id);
2340        }
2341    }
2342
2343    /// The tail of the graph after a trustworthy tally: fold losers, review,
2344    /// gate, merge, and persist.
2345    async fn finish_after_tally(&mut self) -> Result<()> {
2346        // Before the losers are folded: their branches are what the graft
2347        // reads from.
2348        self.graft().await?;
2349        if self.resource_halt()? {
2350            return Ok(());
2351        }
2352        self.fold_losers().await?;
2353        // Before review starts, and again right before the gate: a run's
2354        // review rounds can themselves take long enough for the base to move
2355        // a second time, and the gate is the one node whose "green" gets
2356        // acted on.
2357        self.sync_to_base().await?;
2358        if self.hold_base_sync().await? || self.resource_halt()? {
2359            return Ok(());
2360        }
2361        if self.state.status == RunStatus::AlreadyInBase {
2362            return Ok(());
2363        }
2364        self.review_loop().await?;
2365        if self.resource_halt()? {
2366            return Ok(());
2367        }
2368        self.sync_to_base().await?;
2369        if self.hold_base_sync().await? || self.resource_halt()? {
2370            return Ok(());
2371        }
2372        if self.state.status == RunStatus::AlreadyInBase {
2373            return Ok(());
2374        }
2375        self.gate().await?;
2376        self.merge().await?;
2377        if self.resource_halt()? {
2378            return Ok(());
2379        }
2380        self.state.save()?;
2381        Ok(())
2382    }
2383
2384    // ---------------------------------------------------------------- prep
2385
2386    /// Run `[worktree] setup` in a seat's fresh worktree. A failure blocks the
2387    /// run with the step and its output in the event; it is never swallowed.
2388    /// During `prep` (`in_prep`) the whole preparation is rolled back, so a
2389    /// resume starts over instead of finding candidates and missing seats.
2390    async fn setup_seat_worktree(&mut self, repo: &Path, wt: &Path, in_prep: bool) -> Result<()> {
2391        let result = match ensure_setup_config(&mut self.state, repo).await {
2392            Ok(()) => worktree_setup::prepare(&self.state.config, repo, wt).await,
2393            Err(e) => Err(e),
2394        };
2395        if let Err(e) = result {
2396            git::worktree_remove_clean(repo, wt).await.ok();
2397            if in_prep {
2398                self.rollback_prep(repo).await;
2399            }
2400            self.state.status = RunStatus::Blocked;
2401            self.state.event(
2402                "setup",
2403                format!("worktree setup failed in {}: {e:#}", wt.display()),
2404            );
2405            self.state.save()?;
2406            return Err(e);
2407        }
2408        Ok(())
2409    }
2410
2411    /// Undo everything `prep` created, so a resume re-runs it whole.
2412    async fn rollback_prep(&mut self, repo: &Path) {
2413        let root = self.state.worktree_root();
2414        for c in std::mem::take(&mut self.state.candidates) {
2415            if matches!(
2416                git::worktree_remove_clean(repo, &c.worktree).await,
2417                Ok(true)
2418            ) {
2419                git::branch_delete(repo, &c.branch).await.ok();
2420            } else {
2421                self.state.candidates.push(c);
2422            }
2423        }
2424        let seats = self.roles.judges.len().max(self.state.config.graph.judges);
2425        for j in 1..=seats {
2426            git::worktree_remove_clean(repo, &root.join(format!("judge-{j}")))
2427                .await
2428                .ok();
2429        }
2430        for k in 1..=self.state.config.graph.advisors {
2431            git::worktree_remove_clean(repo, &root.join(format!("advisor-{k}")))
2432                .await
2433                .ok();
2434        }
2435    }
2436
2437    async fn prep(&mut self) -> Result<()> {
2438        if let Some(reason) = &self.state.prep_failure {
2439            bail!(
2440                "candidate preparation previously failed: {reason}; correct the source before retrying"
2441            );
2442        }
2443        if self
2444            .state
2445            .prep_complete
2446            .unwrap_or(!self.state.candidates.is_empty())
2447        {
2448            return Ok(());
2449        }
2450        let repo = self.state.repo.clone();
2451        // Includes rollback, config, refs, checkout creation and setup. No
2452        // repository mutation may precede the real shared lock acquisition.
2453        let _coordination = self.coordinate(&repo).await?;
2454        let base = self.state.base_commit.clone();
2455        let plan = self.state.sources.plan(&repo, &base).await?;
2456        self.rollback_prep(&repo).await;
2457        if !self.state.candidates.is_empty() {
2458            bail!(
2459                "preparation rollback retained worktrees; recover their uncommitted work before retrying"
2460            );
2461        }
2462        self.state.prep_complete = Some(false);
2463        self.state.status = RunStatus::Prep;
2464        self.state.save()?;
2465        let repo = self.state.repo.clone();
2466        let start = plan.start.clone().unwrap_or_else(|| base.clone());
2467        let root = self.state.worktree_root();
2468        let labels = blind::assign_labels(self.roles.implementers.len(), self.state.seed);
2469
2470        // The hook is the write-time half of the blindness contract; the
2471        // presentation filter in `blind` is the half that cannot be bypassed.
2472        let hooks_dir = self.state.dir().join("hooks");
2473        if self.state.config.blind.commit_msg_hook {
2474            std::fs::create_dir_all(&hooks_dir)
2475                .with_context(|| format!("create {}", hooks_dir.display()))?;
2476            let script = blind::commit_msg_hook(&self.state.config.blind.strip_lines);
2477            let path = hooks_dir.join("commit-msg");
2478            std::fs::write(&path, script).with_context(|| format!("write {}", path.display()))?;
2479            make_executable(&path)?;
2480            // Ref-counted rather than a plain idempotent set: with more than
2481            // one run able to be in flight in the same repository at once
2482            // (see `Config::daemon.max_concurrent_runs`), a bare "already
2483            // true?" check cannot tell "another run of mine still needs
2484            // this" from "nobody does", and the run that happens to finish
2485            // first would disable the hook out from under a sibling still
2486            // relying on it.
2487            git::acquire_worktree_config(&repo).await?;
2488            self.state.enabled_worktree_config = true;
2489        }
2490
2491        // `[worktree] setup` hides its products per worktree, which needs the
2492        // same per-worktree config the hook does. Held until fold, like it.
2493        ensure_setup_config(&mut self.state, &repo).await?;
2494
2495        for (index, (spec, label)) in self
2496            .roles
2497            .implementers
2498            .clone()
2499            .into_iter()
2500            .zip(labels)
2501            .enumerate()
2502        {
2503            let branch = self.state.branch_for(label);
2504            let worktree = root.join(format!("cand-{label}"));
2505            // Persist intent before Git can leave a partially created tree.
2506            // Candidate registration follows setup, so it cannot inventory
2507            // crashes or refusals during creation itself.
2508            self.state.event(
2509                "prep-intent",
2510                serde_json::json!({
2511                    "branch": branch,
2512                    "worktree": worktree,
2513                    "start": start,
2514                    "picks": plan.picks,
2515                })
2516                .to_string(),
2517            );
2518            self.state.save()?;
2519            git::worktree_add_branch(&repo, &worktree, &branch, &start).await?;
2520            if self.state.config.blind.commit_msg_hook {
2521                git::set_worktree_hooks_path(&worktree, &hooks_dir).await?;
2522            }
2523            git::local_exclude(&worktree, "/.magi/").await?;
2524            if let Err(e) = worktree_setup::prepare(&self.state.config, &repo, &worktree).await {
2525                // Nothing of this prep may survive: a resume must start over,
2526                // not find half the candidates and skip the rest.
2527                git::worktree_remove_clean(&repo, &worktree).await.ok();
2528                git::branch_delete(&repo, &branch).await.ok();
2529                self.rollback_prep(&repo).await;
2530                self.state.status = RunStatus::Blocked;
2531                self.state
2532                    .event("prep", format!("worktree setup failed: {e:#}"));
2533                self.state.prep_failure = Some(format!("worktree setup failed: {e:#}"));
2534                self.state.save()?;
2535                return Err(e);
2536            }
2537            self.state.candidates.push(Candidate {
2538                index,
2539                label,
2540                agent: spec.id.clone(),
2541                branch,
2542                worktree,
2543                summary: String::new(),
2544                stat: String::new(),
2545                files: 0,
2546                commits: 0,
2547                empty: false,
2548                failed: None,
2549                verified_noop: None,
2550                duration_ms: 0,
2551                folded: false,
2552            });
2553            self.state.save()?;
2554            for pick in &plan.picks {
2555                if let Err(e) = git::cherry_pick(&self.state.candidates[index].worktree, pick).await
2556                {
2557                    self.state.status = RunStatus::Blocked;
2558                    self.state.prep_failure = Some(format!("cannot apply referenced commit: {e}"));
2559                    self.state
2560                        .event("prep", format!("cannot apply referenced commit: {e}"));
2561                    self.state.save()?;
2562                    return Err(e);
2563                }
2564            }
2565        }
2566
2567        for j in 1..=self.roles.judges.len() {
2568            let wt = root.join(format!("judge-{j}"));
2569            if !wt.exists() {
2570                git::worktree_add_detached(&repo, &wt, &base).await?;
2571                self.setup_seat_worktree(&repo, &wt, true).await?;
2572            }
2573        }
2574
2575        // Disposable, detached checkouts for the design-deliberation stage's
2576        // advisor seats — the same shape as the judges' above, at the same
2577        // base commit, since advisors also only ever read. Sized off the
2578        // configured count directly rather than a resolved roster: unlike
2579        // `implementers`/`judges`/`reviewers`, advisor seats are resolved
2580        // lazily inside `advise` itself (see `Config::advisors`'s doc), so
2581        // `prep` has no `ResolvedRoles` field to read a count from here.
2582        if self.state.config.graph.advise {
2583            for k in 1..=self.state.config.graph.advisors {
2584                let wt = root.join(format!("advisor-{k}"));
2585                if !wt.exists() {
2586                    git::worktree_add_detached(&repo, &wt, &base).await?;
2587                    self.setup_seat_worktree(&repo, &wt, true).await?;
2588                }
2589            }
2590        }
2591
2592        // A judge cannot tell it is looking at its own patch — the seats keep
2593        // separate conversations — but a panel that shares agents with the
2594        // field is less independent than it looks, and that is worth saying out
2595        // loud once per run rather than leaving it in the config.
2596        let authors: Vec<&str> = self
2597            .roles
2598            .implementers
2599            .iter()
2600            .map(|a| a.id.as_str())
2601            .collect();
2602        let overlap: Vec<String> = self
2603            .roles
2604            .judges
2605            .iter()
2606            .enumerate()
2607            .filter(|(_, j)| authors.contains(&j.id.as_str()))
2608            .map(|(i, j)| format!("judge {} = {}", i + 1, j.id))
2609            .collect();
2610        if !overlap.is_empty() {
2611            let note = format!(
2612                "{} also authored a candidate; blind, but the panel is less \
2613                 independent than {} distinct agents would be",
2614                overlap.join(", "),
2615                self.roles.judges.len()
2616            );
2617            self.state.event("prep", note);
2618        }
2619
2620        self.state.event(
2621            "prep",
2622            format!(
2623                "{} candidates, {} judges, base {} ({})",
2624                self.state.candidates.len(),
2625                self.roles.judges.len(),
2626                &self.state.base_commit[..7.min(self.state.base_commit.len())],
2627                self.state.base_branch
2628            ),
2629        );
2630        self.state.prep_complete = Some(true);
2631        self.state.status = RunStatus::Implementing;
2632        self.state.save()?;
2633        Ok(())
2634    }
2635
2636    // -------------------------------------------------------------- advise
2637
2638    /// The design-deliberation stage: independent, read-only advisor seats
2639    /// each sketch a design before any implementer touches the repository,
2640    /// and (when at least one produced a usable proposal) a synthesis seat
2641    /// blends them into a brief `implement` carries in every candidate's
2642    /// prompt.
2643    ///
2644    /// `[graph] advise` is the on/off switch, on by default; `[graph]
2645    /// advisors` is the proposal count. Everything here is best-effort and
2646    /// non-fatal to the run: a misconfigured `[roles] advisors`, a roster
2647    /// that cannot reach quota, or a synthesis seat that produced nothing
2648    /// usable all leave `implement` exactly as it was before this stage
2649    /// existed — the task instruction alone — rather than failing the whole
2650    /// competition over an enrichment stage. Every outcome is still recorded
2651    /// as an event, so a run that got nothing from this stage says why.
2652    ///
2653    /// [`RunState::advise_attempted`] is this node's idempotency marker, the
2654    /// same role [`RunState::judge_skipped`] plays for `judge`: without it a
2655    /// resumed run whose stage failed would re-run it, and re-spend the
2656    /// agent calls, on every reentry before `implement`.
2657    ///
2658    /// Also skipped once any candidate shows implementation progress — the
2659    /// exact predicate `implement` itself uses to decide a candidate is no
2660    /// longer "todo" (see its own `todo` filter). `advise_attempted` alone
2661    /// is not enough: a run created by an older binary that predates this
2662    /// field deserializes it as `false` (`#[serde(default)]`), so resuming
2663    /// an already-`Implementing`-or-later run under this build would
2664    /// otherwise walk straight back through `prep` (a no-op once candidates
2665    /// exist) into this node and spawn every advisor seat against worktrees
2666    /// `prep` never recreated — after implementation has already started,
2667    /// which is exactly the invariant this stage exists to guarantee.
2668    async fn advise(&mut self) -> Result<()> {
2669        let implement_untouched = self
2670            .state
2671            .candidates
2672            .iter()
2673            .all(|c| c.commits == 0 && c.failed.is_none() && !c.empty);
2674        if !self.state.config.graph.advise || self.state.advise_attempted {
2675            return Ok(());
2676        }
2677        if !implement_untouched {
2678            self.state.event(
2679                "advise",
2680                "skipping the design-deliberation stage: at least one \
2681                 candidate already shows implementation progress, so this \
2682                 run is past the point the stage exists to run before"
2683                    .to_owned(),
2684            );
2685            self.state.advise_attempted = true;
2686            self.state.save()?;
2687            return Ok(());
2688        }
2689        let run_id = self.state.id.clone();
2690        let prompts = self.state.config.prompts.clone();
2691        let instruction = self.state.instruction.clone();
2692        let language = self.state.config.graph.language.clone();
2693        let root = self.state.worktree_root();
2694        let n = self.state.config.graph.advisors;
2695        let where_recorded = self.state.dir().join("run.json");
2696
2697        let seats = match self.state.config.advisors() {
2698            Ok(seats) if !seats.is_empty() => seats,
2699            Ok(_) => {
2700                self.state.event(
2701                    "advise",
2702                    format!(
2703                        "[graph] advisors is 0; skipping the design-deliberation \
2704                         stage and continuing without a synthesis brief (see {})",
2705                        where_recorded.display()
2706                    ),
2707                );
2708                self.state.advise_attempted = true;
2709                self.state.save()?;
2710                return Ok(());
2711            }
2712            Err(e) => {
2713                self.state.event(
2714                    "advise",
2715                    format!(
2716                        "could not resolve advisor seats ({e:#}); continuing \
2717                         without a design-deliberation brief (see {})",
2718                        where_recorded.display()
2719                    ),
2720                );
2721                self.state.advise_attempted = true;
2722                self.state.save()?;
2723                return Ok(());
2724            }
2725        };
2726
2727        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2728        let artifacts = agent::artifacts_dir(&self.state.dir());
2729        let worktrees: Vec<PathBuf> = (1..=n).map(|k| root.join(format!("advisor-{k}"))).collect();
2730
2731        let mut jobs = Vec::new();
2732        for (i, spec) in seats.iter().cloned().enumerate() {
2733            let seat_key = format!("advisor-{}", i + 1);
2734            let seat = self.seat(&seat_key, &spec.id);
2735            jobs.push(SeatJob {
2736                prompt: prompt::advisor(&instruction, i + 1, seats.len(), &language),
2737                spec,
2738                seat,
2739                cwd: worktrees[i % worktrees.len()].clone(),
2740                timeout,
2741                allow_write: false,
2742                sessions: false,
2743                artifacts: artifacts.clone(),
2744                stem: seat_key,
2745                handover: None,
2746            });
2747        }
2748
2749        self.state.event(
2750            "advise",
2751            format!(
2752                "{} advisor seat(s) sketching a design in parallel",
2753                jobs.len()
2754            ),
2755        );
2756        let mut quota_losses = Vec::new();
2757        let cache = self.state.config.cache_dir();
2758        let ctx = WaveCtx {
2759            carry_seats: false,
2760            pauses: Some((&self.pause, &self.interrupt)),
2761            run: &run_id,
2762            node: "advise",
2763            prompts: &prompts,
2764            cache: cache.as_deref(),
2765            round: None,
2766        };
2767        let advisor_roster = self.state.config.advisor_roster().unwrap_or_default();
2768        let results = ask_json_wave::<Proposal>(
2769            jobs,
2770            Arc::clone(&self.sem),
2771            self.state.config.graph.retries,
2772            &advisor_roster,
2773            &ctx,
2774            &mut quota_losses,
2775            &mut self.state,
2776            &|p: &Proposal| p.validate(),
2777        )
2778        .await;
2779        if self.state.resource_block.is_some() {
2780            return self.stop_for_resource();
2781        }
2782        self.state.quota.extend(quota_losses);
2783
2784        let mut records = Vec::with_capacity(results.len());
2785        for (i, (seat, res, _attempts)) in results.into_iter().enumerate() {
2786            let agent_id = seat.agent.clone();
2787            self.state.seats.insert(seat.key.clone(), seat);
2788            match res {
2789                Ok((proposal, out)) => {
2790                    self.state
2791                        .event("advise", format!("advisor-{} proposed a design", i + 1));
2792                    records.push(advise::AdvisorRecord::proposed(
2793                        i + 1,
2794                        agent_id,
2795                        proposal,
2796                        out.duration_ms,
2797                    ));
2798                }
2799                Err(e) => {
2800                    self.state.event(
2801                        "advise",
2802                        format!("advisor-{} produced no usable proposal: {e:#}", i + 1),
2803                    );
2804                    records.push(advise::AdvisorRecord::failed(
2805                        i + 1,
2806                        agent_id,
2807                        e.to_string(),
2808                    ));
2809                }
2810            }
2811        }
2812
2813        let mut advice = advise::Advice {
2814            records,
2815            synthesis: None,
2816        };
2817        if advice.proposals().is_empty() {
2818            self.state.event(
2819                "advise",
2820                "no advisor produced a usable proposal; continuing without a \
2821                 synthesis brief"
2822                    .to_owned(),
2823            );
2824        } else {
2825            match self
2826                .synthesize_brief(
2827                    &advice,
2828                    &instruction,
2829                    &language,
2830                    &worktrees[0],
2831                    &artifacts,
2832                    &run_id,
2833                    &prompts,
2834                    cache.as_deref(),
2835                )
2836                .await
2837            {
2838                Ok(Some(text)) => {
2839                    self.state.event(
2840                        "advise",
2841                        "synthesized a design brief for the implementer".to_owned(),
2842                    );
2843                    advice.synthesis = Some(text);
2844                }
2845                Ok(None) => {
2846                    self.state.event(
2847                        "advise",
2848                        "the synthesis seat produced nothing usable; continuing \
2849                         without a design brief"
2850                            .to_owned(),
2851                    );
2852                }
2853                Err(e) => {
2854                    if e.is::<crate::repo_lock::ResourceBlocked>() {
2855                        return Err(e);
2856                    }
2857                    self.state.event(
2858                        "advise",
2859                        format!("could not synthesize a design brief: {e:#}"),
2860                    );
2861                }
2862            }
2863        }
2864        advise::apply_reflection(&mut advice);
2865
2866        self.state.advice = Some(advice);
2867        self.state.advise_attempted = true;
2868        self.state.save()?;
2869        Ok(())
2870    }
2871
2872    /// The synthesis seat: reads every advisor's proposal and blends them
2873    /// into the design brief `advise` stores on [`RunState::advice`]. Split
2874    /// out of [`Runner::advise`] only for readability — it is not called
2875    /// anywhere else.
2876    ///
2877    /// Picked the same way [`crate::talk`]'s standing conversation and
2878    /// [`crate::bump`]'s release-bump decision are: [`agent::pick`], with
2879    /// `[roles] synthesizer` checked first and [`agent::pick`]'s own default
2880    /// order (a claude seat, else the first runnable agent in roster order)
2881    /// used when that field is unset — see `[roles] synthesizer`'s own doc
2882    /// in [`crate::config`] for why a dedicated field exists here at all.
2883    #[allow(clippy::too_many_arguments)]
2884    async fn synthesize_brief(
2885        &mut self,
2886        advice: &advise::Advice,
2887        instruction: &str,
2888        language: &str,
2889        cwd: &Path,
2890        artifacts: &Path,
2891        run_id: &str,
2892        prompts: &Prompts,
2893        cache: Option<&Path>,
2894    ) -> Result<Option<String>> {
2895        let chain = agent::pick_chain(
2896            &self.state.config.agents,
2897            self.state.config.roles.synthesizer.as_ref(),
2898            &agent::installed,
2899            "synthesizer",
2900        )?;
2901        let proposals = advice.proposals();
2902        let mut prompt = prompt::with_overlay(
2903            prompt::synthesize_brief(instruction, &proposals, language),
2904            prompts.overlay("advise"),
2905        );
2906        if cache.is_some() {
2907            // This seat never writes, so it is never handed `CARGO_TARGET_DIR`
2908            // below — see `prompt::build_cache_note`'s doc for why telling a
2909            // read-only seat to build through the shared cache is exactly how
2910            // a sandbox's write refusal gets misread as a defect.
2911            prompt.push('\n');
2912            prompt.push_str(&prompt::build_cache_note("advise", false));
2913        }
2914        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge.max(1));
2915        // Each id is tried once, in order; a quota hit, error or unusable
2916        // answer moves to the next. The seat is single-turn (`sessions:
2917        // false`) and the prompt is the whole context, so a fallback agent
2918        // needs nothing carried over.
2919        let mut last = None;
2920        for (n, spec) in chain.iter().enumerate() {
2921            if n > 0 {
2922                self.state
2923                    .event("advise", format!("synthesis falling back to {}", spec.id));
2924            }
2925            let mut seat = self.seat("advise-synthesis", &spec.id);
2926            let wait = self.repository_wait();
2927            let outcome = wait
2928                .scope(agent::invoke(
2929                    spec,
2930                    &mut seat,
2931                    &Invocation {
2932                        coordination: self.state.config.coordination.as_ref(),
2933                        fetched_base: None,
2934                        cwd,
2935                        prompt: &prompt,
2936                        timeout,
2937                        allow_write: false,
2938                        sessions: false,
2939                        artifacts,
2940                        stem: &if n == 0 {
2941                            "advise-synthesis".to_owned()
2942                        } else {
2943                            format!("advise-synthesis-{}", spec.id)
2944                        },
2945                        run: run_id,
2946                        node: "advise",
2947                        cache_dir: None,
2948                        attachments: &[],
2949                        writable: &[],
2950                    },
2951                ))
2952                .await;
2953            if outcome.is_ok() {
2954                self.state.seats.insert(seat.key.clone(), seat);
2955            }
2956            let advance = agent::chain_advances(&outcome);
2957            last = Some(outcome);
2958            if !advance {
2959                break;
2960            }
2961        }
2962        // Exhausted: the last attempt's result is what a single failed seat
2963        // would have produced.
2964        let out = last.expect("a chain holds at least one agent")?;
2965        if !out.usable() {
2966            return Ok(None);
2967        }
2968        let text =
2969            verdict::section(&out.text, "synthesis").unwrap_or_else(|| out.text.trim().to_owned());
2970        Ok((!text.trim().is_empty()).then_some(text))
2971    }
2972
2973    // ----------------------------------------------------------- implement
2974
2975    async fn implement(&mut self) -> Result<()> {
2976        self.persist_candidates().await?;
2977        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
2978        // agent files with `magi task add` name the run that paid for it. The
2979        // prompt overlay is cloned alongside it because the waves borrow it
2980        // while `self` is mutably borrowed by the node's own bookkeeping.
2981        let run_id = self.state.id.clone();
2982        let prompts = self.state.config.prompts.clone();
2983        let todo: Vec<usize> = self
2984            .state
2985            .candidates
2986            .iter()
2987            .enumerate()
2988            .filter(|(_, c)| {
2989                (c.commits == 0 || self.state.interrupted_implementations.contains(&c.label))
2990                    && c.failed.is_none()
2991                    && !c.empty
2992            })
2993            .map(|(i, _)| i)
2994            .collect();
2995        if todo.is_empty() {
2996            return self.after_implement();
2997        }
2998        self.state.status = RunStatus::Implementing;
2999
3000        let language = self.state.config.graph.language.clone();
3001        let github_language = self.state.config.graph.github_language.clone();
3002        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
3003        let sessions = self.state.config.graph.sessions;
3004        let artifacts = agent::artifacts_dir(&self.state.dir());
3005        // The design-deliberation stage's blended brief, when `advise` found
3006        // one — carried into every implementer's prompt the same way
3007        // regardless of which candidate it is.
3008        let brief = self
3009            .state
3010            .advice
3011            .as_ref()
3012            .and_then(|a| a.synthesis.as_deref())
3013            .map(str::to_owned);
3014        let attachments = self.state.attachments.clone();
3015
3016        let mut jobs = Vec::new();
3017        for &i in &todo {
3018            let (index, label, worktree) = {
3019                let c = &self.state.candidates[i];
3020                (c.index, c.label, c.worktree.clone())
3021            };
3022            let spec = self.roles.implementers[index].clone();
3023            let seat_key = format!("impl-{label}");
3024            let seat = self.seat(&seat_key, &spec.id);
3025            let instruction = seeded_instruction(&self.state);
3026            jobs.push(SeatJob {
3027                spec,
3028                seat,
3029                prompt: prompt::implement_in(
3030                    &instruction,
3031                    &worktree.to_string_lossy(),
3032                    &language,
3033                    &github_language,
3034                    brief.as_deref(),
3035                    &attachments,
3036                ),
3037                cwd: worktree,
3038                timeout,
3039                allow_write: true,
3040                sessions,
3041                artifacts: artifacts.clone(),
3042                stem: format!("impl-{label}"),
3043                handover: None,
3044            });
3045        }
3046
3047        self.state.event(
3048            "implement",
3049            format!("{} candidates in parallel", jobs.len()),
3050        );
3051        // Kept so a seat whose CLI hung up can be asked again from the same
3052        // job: `wave` consumes what it is given. Mutable so `resume_seat_handovers`
3053        // can update a seat's own entry once a fallback agent takes it over —
3054        // `resume_unconfirmed_commands`, which reads `sent` afterward, must see
3055        // whichever agent actually answered, not the one that quota'd out.
3056        let mut sent = jobs.clone();
3057        let cache = self.state.config.cache_dir();
3058        let ctx = WaveCtx {
3059            carry_seats: false,
3060            pauses: Some((&self.pause, &self.interrupt)),
3061            run: &run_id,
3062            node: "implement",
3063            prompts: &prompts,
3064            cache: cache.as_deref(),
3065            round: None,
3066        };
3067        let mut results = wave(jobs, Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
3068        if !wave_blocked(&results) {
3069            self.resume_undelivered(&mut results, &sent, &prompts, &run_id)
3070                .await;
3071            self.resume_seat_handovers(&mut results, &mut sent, &prompts, &run_id)
3072                .await?;
3073            self.resume_unconfirmed_commands(&mut results, &sent, &prompts, &run_id)
3074                .await;
3075        }
3076        // Do not finalize failed candidates when the entire wave needs machine
3077        // recovery: leaving them pending resumes this node with its work intact.
3078        if !results.is_empty()
3079            && results
3080                .iter()
3081                .all(|(_, _, out)| matches!(out, AgentOutcome::Quota(_)))
3082        {
3083            let losses: Vec<_> = results
3084                .iter()
3085                .map(|(_, seat, out)| {
3086                    let AgentOutcome::Quota(o) = out else {
3087                        unreachable!()
3088                    };
3089                    QuotaLoss::from_report(seat.key.clone(), "implement", o.quota.as_ref())
3090                })
3091                .collect();
3092            self.state.quota.extend(losses.clone());
3093            for (_, seat, _) in results {
3094                self.state.seats.insert(seat.key.clone(), seat);
3095            }
3096            for &i in &todo {
3097                let label = self.state.candidates[i].label;
3098                self.state.interrupted_implementations.insert(label);
3099                self.state
3100                    .candidate_preservation
3101                    .insert(label, "machine outage awaiting persistence".into());
3102            }
3103            // Preserve partial edits too, while keeping every interrupted seat
3104            // pending even when its tree already contains an inherited patch.
3105            block_machine_outage(&mut self.state, "implement", &losses);
3106            self.state.save()?;
3107            self.persist_candidates().await?;
3108            for &i in &todo {
3109                self.state
3110                    .interrupted_implementations
3111                    .insert(self.state.candidates[i].label);
3112            }
3113            return self.stop_for_resource();
3114        }
3115        let resource_blocked = self.state.resource_block.is_some() || wave_blocked(&results);
3116
3117        let selected_failed = self.state.config.graph.selected_implementer.is_some()
3118            && results
3119                .iter()
3120                .any(|(_, _, out)| FailClass::of(out).is_some());
3121        for (&i, (_wi, seat, out)) in todo.iter().zip(results) {
3122            if matches!(out, AgentOutcome::ResourceBlocked(_)) {
3123                // Executed turns interrupted by disk pressure already carry
3124                // a preservation barrier from wave. Admission-only waits do
3125                // not. Keep the interrupted seat's session for recovery too.
3126                if self
3127                    .state
3128                    .candidate_preservation
3129                    .contains_key(&self.state.candidates[i].label)
3130                {
3131                    self.state.seats.insert(seat.key.clone(), seat);
3132                }
3133                continue;
3134            }
3135            let seat_key = seat.key.clone();
3136            // A quota fallback (`resume_seat_handovers`) may have handed this
3137            // seat to a different agent than the one `prep` recorded on the
3138            // candidate; the stats tables and any later fixer-defaults-to-
3139            // winner's-author lookup must credit whoever actually answered —
3140            // unless every fallback also quota'd out, in which case nobody
3141            // actually answered and crediting the last agent tried would
3142            // erase every earlier agent's own quota loss from the stats
3143            // tables instead of just this one seat's.
3144            let agent = seat.agent.clone();
3145            let exhausted_the_fallback_chain = FailClass::of(&out).is_some();
3146            self.state.seats.insert(seat.key.clone(), seat);
3147            let label = self.state.candidates[i].label;
3148            self.state.interrupted_implementations.remove(&label);
3149
3150            let (summary, duration, failed, verified_claim) = match out {
3151                AgentOutcome::Ok(o) => {
3152                    let text = verdict::section(&o.text, "summary").unwrap_or(o.text.clone());
3153                    let failed = (!o.usable()).then(|| {
3154                        if o.timed_out {
3155                            "agent timed out".to_owned()
3156                        } else {
3157                            format!(
3158                                "agent exited with {}",
3159                                crate::notices::exit_status(o.exit_code)
3160                            )
3161                        }
3162                    });
3163                    let verified_claim = verified_noop_claim(failed.is_none(), &o.commands, &text);
3164                    (text, o.duration_ms, failed, verified_claim)
3165                }
3166                // Left un-resumed by `resume_undelivered` (a dirty tree
3167                // already rescues the work, or there was no session left to
3168                // resume into) — reported like the ordinary failure it is,
3169                // never as if `o.text` (the CLI's raw error JSON) were an
3170                // answer.
3171                AgentOutcome::Dropped(o) => {
3172                    let why = o
3173                        .dropped
3174                        .as_ref()
3175                        .map(|d| d.why.as_str())
3176                        .unwrap_or("the CLI ended the stream without delivering its answer");
3177                    (
3178                        String::new(),
3179                        o.duration_ms,
3180                        Some(format!("the CLI dropped the stream ({why})")),
3181                        None,
3182                    )
3183                }
3184                AgentOutcome::Quota(o) => {
3185                    self.state.quota.push(QuotaLoss::from_report(
3186                        seat_key,
3187                        "implement",
3188                        o.quota.as_ref(),
3189                    ));
3190                    (
3191                        String::new(),
3192                        o.duration_ms,
3193                        Some("rate limited (quota); produced no change".to_owned()),
3194                        None,
3195                    )
3196                }
3197                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
3198                    (String::new(), 0, Some(e), None)
3199                }
3200            };
3201
3202            // Save every completed turn before any fallible Git operation. A failed
3203            // rescue must not erase the reply or prevent collection of other seats.
3204            let c = &mut self.state.candidates[i];
3205            if !exhausted_the_fallback_chain {
3206                c.agent = agent;
3207            }
3208            c.summary = blind::sanitize_prose(&summary, &self.state.config.blind);
3209            c.duration_ms = duration;
3210            c.failed = failed;
3211            c.verified_noop = verified_claim;
3212            self.state
3213                .candidate_preservation
3214                .insert(label, "awaiting persistence".to_owned());
3215        }
3216        self.state.save()?;
3217        self.persist_candidates().await?;
3218
3219        if resource_blocked {
3220            return self.stop_for_resource();
3221        }
3222        if selected_failed {
3223            self.state.status = RunStatus::Blocked;
3224            self.state.event("implement", "selected implementer failed or hit quota; holding without fallback, review or publication");
3225            self.state.save()?;
3226            bail!(
3227                "selected implementer unavailable or quota-limited; restore availability and explicitly release the task"
3228            );
3229        }
3230        self.after_implement()
3231    }
3232
3233    /// Retry only persistence after a completed implementation turn.
3234    async fn persist_candidates(&mut self) -> Result<()> {
3235        let labels: Vec<_> = self.state.candidate_preservation.keys().copied().collect();
3236        for label in labels {
3237            let i = self
3238                .state
3239                .candidates
3240                .iter()
3241                .position(|c| c.label == label)
3242                .context("pending candidate is missing")?;
3243            let c = self.state.candidates[i].clone();
3244            let interrupted = self.state.interrupted_implementations.contains(&label);
3245            let result = async {
3246                let wait = crate::repo_lock::WaitPolicy::new(
3247                    Duration::from_secs(self.state.config.graph.timeout_resource_wait),
3248                    || false,
3249                );
3250                let _coordination =
3251                    coordinate_operation(&mut self.state, &c.worktree, &wait).await?;
3252                // Best effort evidence requires no index writes. A successful snapshot
3253                // never authorizes removal of a tree whose rescue failed.
3254                if let Err(e) = git::snapshot_uncommitted(
3255                    &c.worktree,
3256                    &self
3257                        .state
3258                        .dir()
3259                        .join("artifacts")
3260                        .join(format!("candidate-{label}-recovery")),
3261                )
3262                .await
3263                {
3264                    self.state.event(
3265                        "preserve",
3266                        format!("candidate {label} snapshot incomplete: {e:#}"),
3267                    );
3268                }
3269                let rescued = git::rescue_commit(
3270                    &c.worktree,
3271                    &format!("magi: candidate {label} (uncommitted work)"),
3272                )
3273                .await?;
3274                self.state.note_withheld("implement", &rescued.withheld);
3275                let base = &self.state.base_commit;
3276                let commits = git::commits_ahead(&c.worktree, base, "HEAD").await?;
3277                let patch = git::diff(&c.worktree, base, "HEAD").await?;
3278                let stat = git::diff_stat(&c.worktree, base, "HEAD").await?;
3279                let files = git::changed_files(&c.worktree, base, "HEAD").await?.len();
3280                write_artifact(&self.state, &format!("cand-{label}.patch"), &patch)?;
3281                if c.empty && commits > 0 && !patch.trim().is_empty()
3282                    && (self.state.judge_skipped || !self.state.judgements.is_empty()
3283                        || !self.state.deliberation.is_empty() || !self.state.votes.is_empty()
3284                        || self.state.tally.is_some())
3285                {
3286                    // Keep the old classification and pending marker so every
3287                    // retry stops too. Retain decisions as recovery evidence;
3288                    // only a supported recovery operation may replace them.
3289                    bail!("candidate {label} changes the judged candidate set; supported recovery and fresh judging are required before continuing");
3290                }
3291                let c = &mut self.state.candidates[i];
3292                c.commits = commits;
3293                c.stat = stat;
3294                c.files = files;
3295                let no_changes = commits == 0 || patch.trim().is_empty();
3296                c.empty = no_changes && !interrupted;
3297                if interrupted && no_changes {
3298                    c.failed = None;
3299                    c.verified_noop = None;
3300                }
3301                if !c.empty {
3302                    c.failed = None;
3303                    c.verified_noop = None;
3304                }
3305                self.state.event(
3306                    "preserve",
3307                    format!("candidate {label}: {files} files, {commits} commits"),
3308                );
3309                Ok::<_, anyhow::Error>(())
3310            }
3311            .await;
3312            match result {
3313                Ok(()) => {
3314                    self.state.candidate_preservation.remove(&label);
3315                    if !interrupted || self.state.candidates[i].files > 0 {
3316                        self.state.interrupted_implementations.remove(&label);
3317                    }
3318                }
3319                Err(e) => {
3320                    let reason = format!("candidate {label} preservation failed: {e:#}");
3321                    self.state.event("preserve", &reason);
3322                    self.state.candidate_preservation.insert(label, reason);
3323                }
3324            }
3325            if self.state.candidate_preservation.is_empty()
3326                && self.state.status == RunStatus::Blocked
3327                && self.state.candidates.iter().all(|c| c.failed.is_none())
3328            {
3329                // Persist the recovered phase with the cleared pending marker;
3330                // a crash between saves must not turn this into a selected-seat
3331                // failure hold on the next resume.
3332                self.state.status = RunStatus::Implementing;
3333                self.state.event(
3334                    "preserve",
3335                    "candidate persistence recovered; implementation can continue",
3336                );
3337            }
3338            self.state.save()?;
3339        }
3340        if !self.state.candidate_preservation.is_empty() {
3341            self.state.status = RunStatus::Blocked;
3342            self.state.save()?;
3343            bail!(
3344                "candidate preservation incomplete; retain worktrees and artifacts, repair persistence and resume this run"
3345            );
3346        }
3347        Ok(())
3348    }
3349
3350    /// Ask again, once, for work a CLI did and then failed to hand over.
3351    ///
3352    /// [`agent::dropped_stream`] recognises the one shape observed: an error
3353    /// status with an empty response and a usage report showing output tokens,
3354    /// i.e. **billed work with nothing delivered**. Run 26c7's candidate B was
3355    /// seven minutes and 14,267 output tokens that arrived as an empty
3356    /// candidate, because `agy`'s own subscriber fell behind and hung up.
3357    ///
3358    /// Two conditions, and both matter:
3359    ///
3360    /// - **Only when the tree is untouched.** Often the agent has already
3361    ///   written its files and only the closing message was lost; the rescue
3362    ///   commit below picks that up and there is nothing to ask for. Re-asking
3363    ///   then would pay for a second implementation of work already on disk.
3364    /// - **Once.** A CLI that drops one stream can drop the next, and this
3365    ///   node is the most expensive in the graph.
3366    ///
3367    /// The re-ask is a resume, not a re-run: `has_context` is true because the
3368    /// dropped reply still carried its `conversation_id`, so the seat is asked
3369    /// to finish what it was doing rather than sent the whole task again. It
3370    /// therefore gets a nudge's budget ([`retry_budget`]) - a quarter of the
3371    /// node's - for the same reason a re-ranked judge does: restating finished
3372    /// work is not the work.
3373    ///
3374    /// Unlike a quota this is worth retrying at all: a rate limit fails the
3375    /// same way until it resets, while an abandoned conversation is still
3376    /// there to be picked up.
3377    async fn resume_undelivered(
3378        &mut self,
3379        results: &mut [(usize, SeatState, AgentOutcome)],
3380        sent: &[SeatJob],
3381        prompts: &Prompts,
3382        run_id: &str,
3383    ) {
3384        for (wi, seat, out) in results.iter_mut() {
3385            let Some(dropped) = (match &*out {
3386                AgentOutcome::Dropped(o) => o.dropped.clone(),
3387                _ => None,
3388            }) else {
3389                continue;
3390            };
3391            let Some(job) = sent.get(*wi) else { continue };
3392            // Already on disk? Then only the closing message was lost.
3393            if !git::is_clean(&job.cwd).await.unwrap_or(true) {
3394                self.state.event(
3395                    "implement",
3396                    format!(
3397                        "{}: the CLI dropped the stream after {} output tokens ({}), but the \
3398                         work is in the tree",
3399                        seat.key, dropped.output_tokens, dropped.why
3400                    ),
3401                );
3402                continue;
3403            }
3404            // The re-ask only makes sense as a resume: `resume_after_drop`
3405            // says nothing about the task, trusting the seat to still hold it.
3406            // Without a session to resume — sessions disabled, or this CLI's
3407            // drop shape happened not to carry a session id — that prompt
3408            // would open a brand-new conversation with no context at all,
3409            // which is worse than leaving this as the ordinary failure it
3410            // already is.
3411            if !has_context(&job.spec, seat, job.sessions) {
3412                self.state.event(
3413                    "implement",
3414                    format!(
3415                        "{}: the CLI dropped the stream after {} output tokens ({}), but there \
3416                         is no session left to resume",
3417                        seat.key, dropped.output_tokens, dropped.why
3418                    ),
3419                );
3420                continue;
3421            }
3422            self.state.event(
3423                "implement",
3424                format!(
3425                    "{}: the CLI dropped the stream after {} output tokens ({}); resuming the \
3426                     conversation",
3427                    seat.key, dropped.output_tokens, dropped.why
3428                ),
3429            );
3430            let mut retry = job.clone();
3431            retry.seat = seat.clone();
3432            retry.prompt = prompt::resume_after_drop(&dropped.why);
3433            retry.timeout = retry_budget(job.timeout, true);
3434            retry.stem = format!("{}-resume", job.stem);
3435            let cache = self.state.config.cache_dir();
3436            let ctx = WaveCtx {
3437                carry_seats: false,
3438                pauses: Some((&self.pause, &self.interrupt)),
3439                run: run_id,
3440                node: "implement",
3441                prompts,
3442                cache: cache.as_deref(),
3443                round: None,
3444            };
3445            let (resumed_seat, resumed) =
3446                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3447            *seat = resumed_seat;
3448            *out = resumed;
3449        }
3450    }
3451
3452    /// Fall an implement seat through to the next untried agent in the
3453    /// implementer roster when it lost to quota — or, since the handover was
3454    /// generalised, to a timeout or an ordinary failure (see [`FailClass`] and
3455    /// [`should_hand_over`] for when a non-quota failure stops the chain), the
3456    /// quota path itself being unchanged — instead of leaving the
3457    /// seat's loss final the moment one agent's account runs dry.
3458    ///
3459    /// Solo runs (`graph.implementers = 1`, `daemon::apply_solo`'s forced shape)
3460    /// are the motivating case: `Config::resolve_roles`'s `implementers`
3461    /// truncates to the single slot rotation picked, so a solo task whose one
3462    /// implementer hits quota mid-run used to have nothing else to try. This
3463    /// walks [`ResolvedRoles::implementer_roster`] instead — the untruncated,
3464    /// unrotated roster — which is the only place the *other* candidates in
3465    /// the machine's roster still exist once `implementers` has been cut down
3466    /// to size.
3467    ///
3468    /// Walks forward from just past the seat's own original position in the
3469    /// roster, never wrapping back to the front: a later candidate slot (say
3470    /// `beta`, the roster's second entry) must fall through to the *next*
3471    /// entry (`gamma`) on its own quota loss, not back to `alpha`, which is
3472    /// almost certainly a different candidate's own agent already — and once
3473    /// the roster's tail is exhausted there is nothing left to fall through
3474    /// to for *this* seat, wrapping or not. Tried by `spec.id`, never the
3475    /// whole [`AgentSpec`]: a roster with the same id named twice must not
3476    /// let this retry that id forever. The loop keeps falling through until
3477    /// an attempt lands something other than `Quota` or the roster's tail
3478    /// runs out of untried ids, at which point the seat is left exactly as
3479    /// `implement`'s own `AgentOutcome::Quota` arm already handles it: one
3480    /// `QuotaLoss` recorded, the candidate failed/empty.
3481    ///
3482    /// `sent` is taken mutably and updated with the fallback agent's spec:
3483    /// `resume_unconfirmed_commands`, which runs after this and also reads
3484    /// `sent`, must see whichever agent actually ended up answering the seat
3485    /// — reading the stale, original spec there would check session
3486    /// eligibility against the wrong CLI and could hand a fallback agent's
3487    /// session id to the agent that just lost the seat to quota.
3488    ///
3489    /// Every fallback gets a fresh [`SeatState`], never the quota'd seat's own
3490    /// — `self.seat` only reuses state when the agent id is unchanged, so
3491    /// handing it a different id already gets this for free. Reusing the old
3492    /// seat would resume a different CLI's session as if it were a
3493    /// continuation of this one.
3494    ///
3495    /// Unlike [`Runner::resume_undelivered`], not gated on a clean worktree:
3496    /// a quota loss cuts an agent off mid-turn, so anything already in the
3497    /// tree is unfinished work, not a completed candidate a re-ask would pay
3498    /// for twice. A dirty tree is rescued into a commit first (the same
3499    /// neutral-identity rescue `implement`'s own outcome loop gives every
3500    /// candidate) so the next agent starts clean.
3501    ///
3502    /// The new agent gets the implementer's full prompt and full
3503    /// `timeout_implement` budget, not `resume_after_drop`'s nudge-sized one:
3504    /// it has no session and no context, and is implementing the task from
3505    /// nothing, unlike a resumed drop which is only restating work already
3506    /// done.
3507    ///
3508    /// Every intermediate `Quota` this loop absorbs is folded into a plain
3509    /// `implement` event, never into `self.state.quota` — that is what
3510    /// `daemon.rs`'s own backoff reads to decide a run's task attempt should
3511    /// go unspent, and a seat that ultimately recovered on its second or
3512    /// third agent is not the stalled panel that check exists to catch. Only
3513    /// the final, unrecovered `Quota` (once the roster runs out) ever reaches
3514    /// `self.state.quota`, via the ordinary `AgentOutcome::Quota` arm the
3515    /// outcome loop already has — this helper never pushes to it itself.
3516    async fn resume_seat_handovers(
3517        &mut self,
3518        results: &mut [(usize, SeatState, AgentOutcome)],
3519        sent: &mut [SeatJob],
3520        prompts: &Prompts,
3521        run_id: &str,
3522    ) -> Result<()> {
3523        let instruction = seeded_instruction(&self.state);
3524        let language = self.state.config.graph.language.clone();
3525        let github_language = self.state.config.graph.github_language.clone();
3526        let brief = self
3527            .state
3528            .advice
3529            .as_ref()
3530            .and_then(|a| a.synthesis.as_deref())
3531            .map(str::to_owned);
3532        let attachments = self.state.attachments.clone();
3533        for (wi, seat, out) in results.iter_mut() {
3534            // Who holds the other candidate seats of this wave right now
3535            // (earlier handovers already written back to `sent`).
3536            let others: BTreeSet<String> = sent
3537                .iter()
3538                .enumerate()
3539                .filter(|(j, _)| j != wi)
3540                .map(|(_, j)| j.spec.id.clone())
3541                .collect();
3542            let Some(job) = sent.get_mut(*wi) else {
3543                continue;
3544            };
3545            // Where the seat's own original agent sits in the roster — the
3546            // fallback walk starts just past here, never at the front, so a
3547            // later candidate slot's quota loss does not fall back onto an
3548            // earlier slot's own agent.
3549            let start = self
3550                .roles
3551                .implementer_roster
3552                .iter()
3553                .position(|s| s.id == job.spec.id)
3554                .unwrap_or(0);
3555            let mut tried: BTreeSet<String> = BTreeSet::from([job.spec.id.clone()]);
3556            let mut fallback_attempt = 0usize;
3557            let mut prev: Option<FailClass> = None;
3558            while let Some(cur) = FailClass::of(&*out) {
3559                if !should_hand_over(prev.as_ref(), &cur) {
3560                    break;
3561                }
3562                let Some(next) =
3563                    pick_successor(&self.roles.implementer_roster, start, &tried, None, &others)
3564                        .cloned()
3565                else {
3566                    break;
3567                };
3568                tried.insert(next.id.clone());
3569                fallback_attempt += 1;
3570
3571                let coordination = self.coordinate(&job.cwd).await?;
3572                if let Ok(r) = git::rescue_commit(
3573                    &job.cwd,
3574                    &format!(
3575                        "magi: candidate {} (uncommitted work before {} fallback)",
3576                        seat.key,
3577                        if cur == FailClass::Quota {
3578                            "quota"
3579                        } else {
3580                            "handover"
3581                        }
3582                    ),
3583                )
3584                .await
3585                {
3586                    self.state.note_withheld("implement", &r.withheld);
3587                }
3588                drop(coordination);
3589
3590                record_handover(
3591                    &mut self.state,
3592                    "implement",
3593                    &seat.key,
3594                    &seat.agent,
3595                    &next.id,
3596                    &cur,
3597                    &fail_reason(&*out),
3598                );
3599                prev = Some(cur.clone());
3600
3601                let new_seat = handover_seat(&seat.key, &next.id, self.state.next_seat_seed());
3602                self.state.seats.insert(seat.key.clone(), new_seat.clone());
3603                // Kept in sync on `sent` itself, not just the local retry: a
3604                // later helper (`resume_unconfirmed_commands`) reads `sent`
3605                // after this one returns and must see whichever agent is now
3606                // occupying the seat, not the one that just quota'd out —
3607                // otherwise it would judge session/continuation eligibility
3608                // by the wrong CLI and could resend a fallback's session id
3609                // to the agent that lost it the seat in the first place.
3610                job.spec = next.clone();
3611                let mut retry = job.clone();
3612                retry.seat = new_seat;
3613                retry.prompt = prompt::implement_in(
3614                    &instruction,
3615                    &job.cwd.to_string_lossy(),
3616                    &language,
3617                    &github_language,
3618                    brief.as_deref(),
3619                    &attachments,
3620                );
3621                retry.stem = format!("{}-{}-{}", job.stem, cur.stem_word(), next.id);
3622                let cache = self.state.config.cache_dir();
3623                let ctx = WaveCtx {
3624                    carry_seats: false,
3625                    pauses: Some((&self.pause, &self.interrupt)),
3626                    run: run_id,
3627                    node: "implement",
3628                    prompts,
3629                    cache: cache.as_deref(),
3630                    round: None,
3631                };
3632                let (fallback_seat, fallback_out) = run_one(
3633                    retry,
3634                    Arc::clone(&self.sem),
3635                    &ctx,
3636                    &mut self.state,
3637                    fallback_attempt,
3638                )
3639                .await;
3640                *seat = fallback_seat;
3641                *out = fallback_out;
3642            }
3643        }
3644        Ok(())
3645    }
3646
3647    /// Ask an implement seat's own CLI to confirm what it started, once, when
3648    /// its reply reported a command whose completion status it never
3649    /// confirmed — see [`has_unconfirmed_command`]'s own doc for exactly what
3650    /// that does and does not mean.
3651    ///
3652    /// The completion contract this task asks for, extended to `implement`
3653    /// with the same signal `continue_fix_report` reads for the fixer,
3654    /// rather than a keyword search over the reply or a hard requirement on
3655    /// `## SUMMARY`'s presence — the shape behind fb35, 9566 and e185, where
3656    /// a candidate's CLI turn ended cleanly while a test run it had started
3657    /// had not. A short, ordinary reply with no `## SUMMARY` and no commands
3658    /// named in it at all is untouched by this: `commands` is empty, so
3659    /// there is nothing to be unconfirmed.
3660    ///
3661    /// Unlike `resume_undelivered`, not gated on the tree being untouched:
3662    /// this is not about recovering edits that might already be on disk, it
3663    /// is about a result the seat itself never vouched for, which resuming
3664    /// asks for regardless of what the tree already holds. Bounded to one
3665    /// attempt for the same reason `resume_undelivered` is — this is the
3666    /// most expensive node in the graph — and a seat that still cannot
3667    /// confirm on that attempt is left as whatever its (possibly still
3668    /// unconfirmed) reply says; this does not invent a new "failed" reason
3669    /// for a candidate that otherwise produced a real, committed change.
3670    async fn resume_unconfirmed_commands(
3671        &mut self,
3672        results: &mut [(usize, SeatState, AgentOutcome)],
3673        sent: &[SeatJob],
3674        prompts: &Prompts,
3675        run_id: &str,
3676    ) {
3677        for (wi, seat, out) in results.iter_mut() {
3678            let AgentOutcome::Ok(o) = &*out else {
3679                continue;
3680            };
3681            if !has_unconfirmed_command(&o.commands) {
3682                continue;
3683            }
3684            let Some(job) = sent.get(*wi) else { continue };
3685            if !has_context(&job.spec, seat, job.sessions) {
3686                self.state.event(
3687                    "implement",
3688                    format!(
3689                        "{}: the reply named a command whose own CLI never confirmed the exit \
3690                         status of, but there is no session left to resume",
3691                        seat.key
3692                    ),
3693                );
3694                continue;
3695            }
3696            self.state.event(
3697                "implement",
3698                format!(
3699                    "{}: the reply named a command whose own CLI never confirmed the exit \
3700                     status of; resuming the conversation",
3701                    seat.key
3702                ),
3703            );
3704            let mut retry = job.clone();
3705            retry.seat = seat.clone();
3706            retry.prompt = prompt::resume_incomplete(
3707                "a command in your last reply had no confirmed exit status",
3708            );
3709            retry.timeout = retry_budget(job.timeout, true);
3710            retry.stem = format!("{}-confirm", job.stem);
3711            let cache = self.state.config.cache_dir();
3712            let ctx = WaveCtx {
3713                carry_seats: false,
3714                pauses: Some((&self.pause, &self.interrupt)),
3715                run: run_id,
3716                node: "implement",
3717                prompts,
3718                cache: cache.as_deref(),
3719                round: None,
3720            };
3721            let (resumed_seat, resumed) =
3722                run_one(retry, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
3723            *seat = resumed_seat;
3724            *out = resumed;
3725        }
3726    }
3727
3728    /// Ask the fixer's own seat again, up to [`MAX_FIX_CONTINUATIONS`] times,
3729    /// when its CLI turn ended cleanly (`AgentOutcome::Ok`) but the reply held
3730    /// no [`FixReport`] — see [`MAX_FIX_CONTINUATIONS`]'s own doc for the run
3731    /// that motivated this.
3732    ///
3733    /// Not the same gap as an unparsable *shape*, which [`ask_json_wave`]'s
3734    /// own nudge loop already covers for judge/review/vote seats, and not a
3735    /// dropped stream, which [`Runner::resume_undelivered`] covers for
3736    /// implement seats: here the CLI turn genuinely finished while the node's
3737    /// own work — the fixer's account of what it did — had not. Gated purely
3738    /// on `extract_json::<FixReport>` having failed on an otherwise-usable
3739    /// reply, never on any wording in it, so a fixer whose valid, first-try
3740    /// `FixReport` happens to mention having waited on a background test is
3741    /// never resumed — the `Ok(report)` branch at the call site returns
3742    /// before this is ever invoked.
3743    ///
3744    /// Same discipline as `resume_undelivered`: a nudge-sized timeout per
3745    /// attempt ([`retry_budget`]), nothing attempted once the session is
3746    /// gone, and a quota hit ends the loop immediately rather than retrying a
3747    /// rate limit that fails the same way again.
3748    async fn continue_fix_report(
3749        &mut self,
3750        mut seat: SeatState,
3751        parse_err: String,
3752        job: &SeatJob,
3753        prompts: &Prompts,
3754        run_id: &str,
3755        round: usize,
3756    ) -> (
3757        SeatState,
3758        Option<FixReport>,
3759        Option<String>,
3760        ContinuationRecord,
3761    ) {
3762        let mut last_err = parse_err;
3763        let mut cumulative_wait_ms = 0u64;
3764        let mut attempts = 0usize;
3765        loop {
3766            if !has_context(&job.spec, &seat, job.sessions) {
3767                self.state.event(
3768                    "fix",
3769                    format!(
3770                        "round {round}: fixer's reply had no adoption report ({last_err}); no \
3771                         session left to resume into"
3772                    ),
3773                );
3774                let outcome = if attempts == 0 {
3775                    ContinuationOutcome::NoSession
3776                } else {
3777                    ContinuationOutcome::Exhausted
3778                };
3779                return (
3780                    seat,
3781                    None,
3782                    Some(format!("unparsable fix report: {last_err}")),
3783                    ContinuationRecord {
3784                        attempts,
3785                        cumulative_wait_ms,
3786                        outcome,
3787                    },
3788                );
3789            }
3790            if attempts >= MAX_FIX_CONTINUATIONS {
3791                self.state.event(
3792                    "fix",
3793                    format!(
3794                        "round {round}: fixer's reply still had no adoption report after \
3795                         {attempts} continuation(s) ({last_err}); giving up"
3796                    ),
3797                );
3798                return (
3799                    seat,
3800                    None,
3801                    Some(format!(
3802                        "unparsable fix report after {attempts} continuation(s): {last_err}"
3803                    )),
3804                    ContinuationRecord {
3805                        attempts,
3806                        cumulative_wait_ms,
3807                        outcome: ContinuationOutcome::Exhausted,
3808                    },
3809                );
3810            }
3811            attempts += 1;
3812            self.state.event(
3813                "fix",
3814                format!(
3815                    "round {round}: fixer's reply had no adoption report ({last_err}); resuming \
3816                     the conversation (attempt {attempts}/{MAX_FIX_CONTINUATIONS})"
3817                ),
3818            );
3819            let mut retry = job.clone();
3820            retry.seat = seat.clone();
3821            retry.prompt = prompt::resume_incomplete(&last_err);
3822            retry.timeout = retry_budget(job.timeout, true);
3823            retry.stem = format!("{}-continue{attempts}", job.stem);
3824            let cache = self.state.config.cache_dir();
3825            let ctx = WaveCtx {
3826                carry_seats: false,
3827                pauses: Some((&self.pause, &self.interrupt)),
3828                run: run_id,
3829                node: "fix",
3830                prompts,
3831                cache: cache.as_deref(),
3832                round: Some(round),
3833            };
3834            let (resumed_seat, resumed_out) = run_one(
3835                retry,
3836                Arc::clone(&self.sem),
3837                &ctx,
3838                &mut self.state,
3839                attempts,
3840            )
3841            .await;
3842            seat = resumed_seat;
3843            match resumed_out {
3844                AgentOutcome::Ok(o) => {
3845                    cumulative_wait_ms += o.duration_ms;
3846                    match verdict::extract_json::<FixReport>(&o.text) {
3847                        Ok(report) if !has_unconfirmed_command(&o.commands) => {
3848                            self.state.event(
3849                                "fix",
3850                                format!(
3851                                    "round {round}: fixer's adoption report recovered after \
3852                                     {attempts} continuation(s)"
3853                                ),
3854                            );
3855                            return (
3856                                seat,
3857                                Some(report),
3858                                None,
3859                                ContinuationRecord {
3860                                    attempts,
3861                                    cumulative_wait_ms,
3862                                    outcome: ContinuationOutcome::Resumed,
3863                                },
3864                            );
3865                        }
3866                        // The report parsed, but this same reply's own
3867                        // CommandEvidence — the identical record `state.jobs`
3868                        // renders — names a command whose CLI never
3869                        // confirmed an exit status. Read together, that is
3870                        // not a resolved answer: keep nudging rather than
3871                        // accept a report standing next to a command the
3872                        // seat's own CLI cannot vouch for.
3873                        Ok(_) => {
3874                            last_err = "the reply parsed, but it reported a command whose own CLI \
3875                                 never confirmed an exit status"
3876                                .to_owned();
3877                        }
3878                        Err(e) => last_err = e.to_string(),
3879                    }
3880                }
3881                AgentOutcome::Quota(o) => {
3882                    cumulative_wait_ms += o.duration_ms;
3883                    let loss = QuotaLoss::from_report(seat.key.clone(), "fix", o.quota.as_ref());
3884                    block_machine_outage(&mut self.state, "fix", std::slice::from_ref(&loss));
3885                    self.state.quota.push(loss);
3886                    self.state.event(
3887                        "fix",
3888                        format!(
3889                            "round {round}: continuation rate limited (quota); not retrying now"
3890                        ),
3891                    );
3892                    return (
3893                        seat,
3894                        None,
3895                        Some("rate limited (quota) while recovering the fix report".to_owned()),
3896                        ContinuationRecord {
3897                            attempts,
3898                            cumulative_wait_ms,
3899                            outcome: ContinuationOutcome::QuotaLost,
3900                        },
3901                    );
3902                }
3903                AgentOutcome::Dropped(o) => {
3904                    cumulative_wait_ms += o.duration_ms;
3905                    let why = o
3906                        .dropped
3907                        .as_ref()
3908                        .map(|d| d.why.as_str())
3909                        .unwrap_or("the CLI ended the stream without delivering its answer");
3910                    last_err = format!("the CLI dropped the stream ({why})");
3911                }
3912                AgentOutcome::Failed(e) => last_err = e,
3913                // The continuation was never started; the report stays lost
3914                // and the caller records that, without any handover.
3915                AgentOutcome::ResourceBlocked(e) => {
3916                    return (
3917                        seat,
3918                        None,
3919                        Some(format!("the continuation was not started: {e}")),
3920                        ContinuationRecord {
3921                            attempts: attempts.saturating_sub(1),
3922                            cumulative_wait_ms,
3923                            outcome: ContinuationOutcome::NoSession,
3924                        },
3925                    );
3926                }
3927            }
3928        }
3929    }
3930
3931    fn after_implement(&mut self) -> Result<()> {
3932        if !self.state.candidate_preservation.is_empty() {
3933            bail!("candidate preservation incomplete; judging is unsafe");
3934        }
3935        // Scan every candidate patch once the set is complete.
3936        if self.state.leaks.is_empty() {
3937            let cfg = self.state.config.blind.clone();
3938            let mut leaks = Vec::new();
3939            for c in &self.state.candidates {
3940                let Some(patch) =
3941                    crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
3942                else {
3943                    continue;
3944                };
3945                leaks.extend(blind::scan(
3946                    &format!("candidate {} patch", c.label),
3947                    &patch,
3948                    &cfg.vendor_tokens,
3949                ));
3950            }
3951            if !leaks.is_empty() {
3952                let summary = leaks
3953                    .iter()
3954                    .map(|l| format!("{}×{} in {}", l.token, l.count, l.site))
3955                    .collect::<Vec<_>>()
3956                    .join(", ");
3957                match cfg.on_leak {
3958                    LeakPolicy::Fail => {
3959                        self.state.status = RunStatus::Failed;
3960                        self.state
3961                            .event("blind", format!("vendor text in a patch: {summary}"));
3962                        self.state.leaks = leaks;
3963                        self.state.save()?;
3964                        self.settle_questions();
3965                        bail!(
3966                            "blind.on_leak = \"fail\" and vendor text reached a \
3967                             judged patch: {summary}"
3968                        );
3969                    }
3970                    LeakPolicy::Redact => self.state.event(
3971                        "blind",
3972                        format!("redacting vendor text for judging: {summary}"),
3973                    ),
3974                    LeakPolicy::Warn => self.state.event(
3975                        "blind",
3976                        format!("vendor text present in a judged patch (shown as-is): {summary}"),
3977                    ),
3978                }
3979                self.state.leaks = leaks;
3980            }
3981        }
3982
3983        if self.state.viable().is_empty() {
3984            if self.state.all_candidates_verified_noop() {
3985                // Every candidate agreed, with evidence the adoption guard
3986                // accepted, that nothing belongs in this worktree. That is
3987                // not the same fact as a candidate that simply failed to
3988                // write anything, and settling it as an ordinary `Failed`
3989                // (see `SCHEMA`'s doc for schema 10) is what let two of
3990                // task 391f's attempts burn a retry each re-discovering the
3991                // same already-landed fix. Terminal either way, so `judge`
3992                // must never run over an empty candidate set — unlike the
3993                // `Failed` branch below this returns `Ok`, not an error:
3994                // nothing here failed.
3995                self.state.status = RunStatus::VerifiedNoop;
3996                self.state.save()?;
3997                self.settle_questions();
3998                return Ok(());
3999            }
4000            self.state.status = RunStatus::Failed;
4001            self.state.save()?;
4002            self.settle_questions();
4003            bail!("no candidate produced a change; nothing to judge");
4004        }
4005        self.state.status = RunStatus::Judging;
4006        self.state.save()?;
4007        Ok(())
4008    }
4009
4010    // --------------------------------------------------------------- judge
4011
4012    async fn judge(&mut self) -> Result<()> {
4013        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4014        // agent files with `magi task add` name the run that paid for it. The
4015        // prompt overlay is cloned alongside it because the waves borrow it
4016        // while `self` is mutably borrowed by the node's own bookkeeping.
4017        let run_id = self.state.id.clone();
4018        let prompts = self.state.config.prompts.clone();
4019        if !self.state.judgements.is_empty() || self.state.judge_skipped {
4020            return Ok(());
4021        }
4022        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4023        if viable.len() == 1 {
4024            // Recorded so this is a one-time event: `judgements` stays empty
4025            // either way, which without this flag is indistinguishable from
4026            // "not yet judged" on the next reentry — and status is left
4027            // untouched, so a later node's conclusion (e.g. `Blocked` after
4028            // the review budget ran out) survives a resume instead of being
4029            // clobbered back to `Judging` by this node running again.
4030            self.state.judge_skipped = true;
4031            self.state.event(
4032                "judge",
4033                format!(
4034                    "only candidate {} produced a change; judging skipped",
4035                    viable[0].label
4036                ),
4037            );
4038            self.state.save()?;
4039            return Ok(());
4040        }
4041        self.state.status = RunStatus::Judging;
4042
4043        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4044        let language = self.state.config.graph.language.clone();
4045        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4046        let sessions = self.state.config.graph.sessions;
4047        let artifacts = agent::artifacts_dir(&self.state.dir());
4048        let root = self.state.worktree_root();
4049        let base_short = short(&self.state.base_commit);
4050
4051        let mut jobs = Vec::new();
4052        let mut orders = Vec::new();
4053        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4054            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4055            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4056            orders.push(order.iter().map(|&k| viable[k].index).collect::<Vec<_>>());
4057            let seat_key = format!("judge-{}", j + 1);
4058            let seat = self.seat(&seat_key, &spec.id);
4059            jobs.push(SeatJob {
4060                prompt: prompt::judge(
4061                    &self.state.instruction,
4062                    &views,
4063                    self.roles.judges.len(),
4064                    &base_short,
4065                    &language,
4066                ),
4067                spec,
4068                seat,
4069                cwd: root.join(format!("judge-{}", j + 1)),
4070                timeout,
4071                allow_write: false,
4072                sessions,
4073                artifacts: artifacts.clone(),
4074                stem: format!("judge-{}", j + 1),
4075                handover: None,
4076            });
4077        }
4078
4079        self.state.event(
4080            "judge",
4081            format!(
4082                "{} judges ranking {} candidates blind",
4083                jobs.len(),
4084                viable.len()
4085            ),
4086        );
4087        let labels_for_check = labels.clone();
4088        let mut quota_losses = Vec::new();
4089        let cache = self.state.config.cache_dir();
4090        let ctx = WaveCtx {
4091            carry_seats: false,
4092            pauses: Some((&self.pause, &self.interrupt)),
4093            run: &run_id,
4094            node: "judge",
4095            prompts: &prompts,
4096            cache: cache.as_deref(),
4097            round: None,
4098        };
4099        let results = ask_json_wave::<Ranking>(
4100            jobs,
4101            Arc::clone(&self.sem),
4102            self.state.config.graph.retries,
4103            &self.roles.judge_roster,
4104            &ctx,
4105            &mut quota_losses,
4106            &mut self.state,
4107            &move |r: &Ranking| r.validate(&labels_for_check),
4108        )
4109        .await;
4110        if self.state.resource_block.is_some() {
4111            return self.stop_for_resource();
4112        }
4113        self.state.quota.extend(quota_losses);
4114
4115        for (j, (seat, res, _attempts)) in results.into_iter().enumerate() {
4116            let agent_id = seat.agent.clone();
4117            self.state.seats.insert(seat.key.clone(), seat);
4118            let mut record = Judgement {
4119                judge: j + 1,
4120                seat: format!("judge-{}", j + 1),
4121                agent: agent_id,
4122                ranking: Vec::new(),
4123                reasons: BTreeMap::new(),
4124                confidence: None,
4125                strengths: BTreeMap::new(),
4126                order: orders[j].clone(),
4127                failed: None,
4128                duration_ms: 0,
4129            };
4130            match res {
4131                Ok((ranking, out)) => {
4132                    record.ranking = ranking.normalized();
4133                    record.reasons = ranking.reasons;
4134                    record.strengths = ranking.strengths;
4135                    record.confidence = ranking.confidence;
4136                    record.duration_ms = out.duration_ms;
4137                    self.state.event(
4138                        "judge",
4139                        format!(
4140                            "judge {} ranked {}",
4141                            j + 1,
4142                            record.ranking.iter().collect::<String>()
4143                        ),
4144                    );
4145                }
4146                Err(e) => {
4147                    record.failed = Some(e.to_string());
4148                    self.state
4149                        .event("judge", format!("judge {} produced no ranking: {e}", j + 1));
4150                }
4151            }
4152            self.state.judgements.push(record);
4153            self.state.save()?;
4154        }
4155        Ok(())
4156    }
4157
4158    // ---------------------------------------------------------- deliberate
4159
4160    async fn deliberate(&mut self) -> Result<()> {
4161        if self.hold_base_sync().await? {
4162            return Ok(());
4163        }
4164        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4165        // agent files with `magi task add` name the run that paid for it. The
4166        // prompt overlay is cloned alongside it because the waves borrow it
4167        // while `self` is mutably borrowed by the node's own bookkeeping.
4168        let run_id = self.state.id.clone();
4169        let prompts = self.state.config.prompts.clone();
4170        let rounds = self.state.config.graph.deliberate_rounds;
4171        // Resume unfinished rounds after an outage, but never reopen a tally.
4172        if self.state.deliberation.len() >= rounds || self.state.tally.is_some() {
4173            return Ok(());
4174        }
4175        let tops: Vec<char> = self
4176            .state
4177            .judgements
4178            .iter()
4179            .filter_map(|j| j.ranking.first().copied())
4180            .collect();
4181        if tops.len() < 2 || tops.iter().all(|t| *t == tops[0]) || rounds == 0 {
4182            if tops.len() >= 2 && tops.iter().all(|t| *t == tops[0]) {
4183                self.state.event(
4184                    "deliberate",
4185                    format!("judges agreed on {} outright; no deliberation", tops[0]),
4186                );
4187            }
4188            self.state.status = RunStatus::Voting;
4189            self.state.save()?;
4190            return Ok(());
4191        }
4192
4193        self.state.status = RunStatus::Deliberating;
4194        self.state.event(
4195            "deliberate",
4196            format!(
4197                "split: first choices were {} — opening {rounds} round(s)",
4198                tops.iter().collect::<String>()
4199            ),
4200        );
4201
4202        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4203        let language = self.state.config.graph.language.clone();
4204        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4205        let sessions = self.state.config.graph.sessions;
4206        let artifacts = agent::artifacts_dir(&self.state.dir());
4207        let root = self.state.worktree_root();
4208        let base_short = short(&self.state.base_commit);
4209
4210        // Judges argue in sequence so that a turn can answer the one before it;
4211        // that is the difference between deliberation and three parallel
4212        // monologues.
4213        for round in self.state.deliberation.len() + 1..=rounds {
4214            let mut turns: Vec<DeliberationTurn> = Vec::new();
4215            let mut round_losses = Vec::new();
4216            let mut asked = 0;
4217            for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4218                if self.state.judgements[j].failed.is_some() {
4219                    continue;
4220                }
4221                let seat_key = format!("judge-{}", j + 1);
4222                let spec = self.occupant(&seat_key, spec);
4223                let mut seat = self.seat(&seat_key, &spec.id);
4224                let transcript = self.transcript(&turns, j);
4225                let build = |context: Option<&str>| {
4226                    prompt::deliberate(
4227                        &self.state.instruction,
4228                        context,
4229                        &transcript,
4230                        round,
4231                        rounds,
4232                        &language,
4233                    )
4234                };
4235                let block = self.candidate_block(&viable, &base_short);
4236                let full = build(Some(&block));
4237                let text = if has_context(&spec, &seat, sessions) {
4238                    build(None)
4239                } else {
4240                    full.clone()
4241                };
4242                let job = SeatJob {
4243                    spec,
4244                    seat: seat.clone(),
4245                    prompt: text,
4246                    cwd: root.join(format!("judge-{}", j + 1)),
4247                    timeout,
4248                    allow_write: false,
4249                    sessions,
4250                    artifacts: artifacts.clone(),
4251                    stem: format!("delib-{round}-judge-{}", j + 1),
4252                    handover: Some(full),
4253                };
4254                let cache = self.state.config.cache_dir();
4255                let ctx = WaveCtx {
4256                    carry_seats: false,
4257                    pauses: Some((&self.pause, &self.interrupt)),
4258                    run: &run_id,
4259                    node: "deliberate",
4260                    prompts: &prompts,
4261                    cache: cache.as_deref(),
4262                    round: None,
4263                };
4264                // A turn is never nudged (`retries` 0): a failed seat is
4265                // handed to the next roster agent, which gets the full
4266                // context. An empty answer is a turn, not a failure.
4267                let mut losses = Vec::new();
4268                asked += 1;
4269                let mut results = ask_wave_with::<String>(
4270                    vec![job],
4271                    Arc::clone(&self.sem),
4272                    0,
4273                    &self.roles.judge_roster,
4274                    &ctx,
4275                    &mut losses,
4276                    &mut self.state,
4277                    &|text: &str| {
4278                        Ok(verdict::section(text, "position").unwrap_or_else(|| text.to_owned()))
4279                    },
4280                )
4281                .await;
4282                if self.state.resource_block.is_some() {
4283                    return self.stop_for_resource();
4284                }
4285                round_losses.extend(losses.iter().cloned());
4286                self.state.quota.extend(losses);
4287                let (updated, res, _) = results.pop().expect("one job in, one result out");
4288                seat = updated;
4289                let agent_id = seat.agent.clone();
4290                self.state.seats.insert(seat.key.clone(), seat);
4291                let body = match res {
4292                    Ok((body, _)) => body,
4293                    // Skip the seat; a CLI's raw error JSON is never read as
4294                    // this judge's position.
4295                    Err(e) => {
4296                        self.state
4297                            .event("deliberate", format!("judge {} skipped: {e}", j + 1));
4298                        continue;
4299                    }
4300                };
4301                let tentative = verdict::extract_json::<Position>(&body)
4302                    .ok()
4303                    .and_then(|p| p.tentative)
4304                    .and_then(|s| s.trim().chars().next())
4305                    .map(|c| c.to_ascii_uppercase());
4306                self.state.event(
4307                    "deliberate",
4308                    format!(
4309                        "round {round}: judge {} now favours {}",
4310                        j + 1,
4311                        tentative.map_or("—".to_owned(), |c| c.to_string())
4312                    ),
4313                );
4314                turns.push(DeliberationTurn {
4315                    judge: j + 1,
4316                    agent: agent_id,
4317                    body: blind::sanitize_prose(&body, &self.state.config.blind),
4318                    tentative,
4319                });
4320            }
4321            if asked > 0 && turns.is_empty() && round_losses.len() == asked {
4322                block_machine_outage(&mut self.state, "deliberate", &round_losses);
4323                return self.stop_for_resource();
4324            }
4325            self.state
4326                .deliberation
4327                .push(DeliberationRound { round, turns });
4328            self.state.save()?;
4329        }
4330
4331        self.state.status = RunStatus::Voting;
4332        self.state.save()?;
4333        Ok(())
4334    }
4335
4336    // ---------------------------------------------------------------- vote
4337
4338    async fn vote(&mut self) -> Result<()> {
4339        if self.hold_base_sync().await? {
4340            return Ok(());
4341        }
4342        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4343        // agent files with `magi task add` name the run that paid for it. The
4344        // prompt overlay is cloned alongside it because the waves borrow it
4345        // while `self` is mutably borrowed by the node's own bookkeeping.
4346        let run_id = self.state.id.clone();
4347        let prompts = self.state.config.prompts.clone();
4348        if !self.state.votes.is_empty() || self.state.tally.is_some() {
4349            return Ok(());
4350        }
4351        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4352        if viable.len() == 1 {
4353            return Ok(());
4354        }
4355        self.state.status = RunStatus::Voting;
4356
4357        let language = self.state.config.graph.language.clone();
4358        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4359        let sessions = self.state.config.graph.sessions;
4360        let artifacts = agent::artifacts_dir(&self.state.dir());
4361        let root = self.state.worktree_root();
4362        let base_short = short(&self.state.base_commit);
4363        let candidates: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4364
4365        let mut jobs = Vec::new();
4366        let mut seats_at = Vec::new();
4367        for (j, spec) in self.roles.judges.clone().into_iter().enumerate() {
4368            if self
4369                .state
4370                .judgements
4371                .get(j)
4372                .is_some_and(|r| r.failed.is_some())
4373            {
4374                continue;
4375            }
4376            let seat_key = format!("judge-{}", j + 1);
4377            let spec = self.occupant(&seat_key, spec);
4378            let seat = self.seat(&seat_key, &spec.id);
4379            let full = self.vote_prompt_full(j, &viable, &language, &candidates, &base_short);
4380            let text = if has_context(&spec, &seat, sessions) {
4381                prompt::final_vote(&viable, &language)
4382            } else {
4383                full.clone()
4384            };
4385            jobs.push(SeatJob {
4386                spec,
4387                seat,
4388                prompt: text,
4389                cwd: root.join(format!("judge-{}", j + 1)),
4390                timeout,
4391                allow_write: false,
4392                sessions,
4393                artifacts: artifacts.clone(),
4394                stem: format!("vote-judge-{}", j + 1),
4395                handover: Some(full),
4396            });
4397            seats_at.push(j);
4398        }
4399
4400        self.state.event(
4401            "vote",
4402            format!(
4403                "collecting {} final votes one by one, privately",
4404                jobs.len()
4405            ),
4406        );
4407        let allowed = viable.clone();
4408        let mut quota_losses = Vec::new();
4409        let cache = self.state.config.cache_dir();
4410        let ctx = WaveCtx {
4411            carry_seats: false,
4412            pauses: Some((&self.pause, &self.interrupt)),
4413            run: &run_id,
4414            node: "vote",
4415            prompts: &prompts,
4416            cache: cache.as_deref(),
4417            round: None,
4418        };
4419        let results = ask_json_wave::<FinalVote>(
4420            jobs,
4421            Arc::clone(&self.sem),
4422            self.state.config.graph.retries,
4423            &self.roles.judge_roster,
4424            &ctx,
4425            &mut quota_losses,
4426            &mut self.state,
4427            &move |v: &FinalVote| match v.label() {
4428                Some(c) if allowed.contains(&c) => Ok(()),
4429                other => bail!("vote {other:?} is not one of {allowed:?}"),
4430            },
4431        )
4432        .await;
4433        if self.state.resource_block.is_some() {
4434            return self.stop_for_resource();
4435        }
4436        self.state.quota.extend(quota_losses);
4437
4438        for (&j, (seat, res, _attempts)) in seats_at.iter().zip(results) {
4439            let agent_id = seat.agent.clone();
4440            self.state.seats.insert(seat.key.clone(), seat);
4441            let initial = self
4442                .state
4443                .judgements
4444                .get(j)
4445                .and_then(|r| r.ranking.first().copied());
4446            let mut record = VoteRecord {
4447                judge: j + 1,
4448                agent: agent_id,
4449                vote: None,
4450                reason: String::new(),
4451                changed: false,
4452            };
4453            match res {
4454                Ok((v, _)) => {
4455                    record.vote = v.label();
4456                    record.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4457                    record.changed = matches!((record.vote, initial), (Some(a), Some(b)) if a != b);
4458                    self.state.event(
4459                        "vote",
4460                        format!(
4461                            "judge {} voted {}{}",
4462                            j + 1,
4463                            record.vote.unwrap_or('?'),
4464                            if record.changed { " (changed)" } else { "" }
4465                        ),
4466                    );
4467                }
4468                Err(e) => {
4469                    self.state
4470                        .event("vote", format!("judge {} cast no vote: {e}", j + 1));
4471                }
4472            }
4473            self.state.votes.push(record);
4474            self.state.save()?;
4475        }
4476        Ok(())
4477    }
4478
4479    // --------------------------------------------------------------- tally
4480
4481    fn tally(&mut self) -> Result<()> {
4482        if self.state.tally.is_some() {
4483            return Ok(());
4484        }
4485        let viable: Vec<char> = self.state.viable().into_iter().map(|c| c.label).collect();
4486        let tops: Vec<char> = self
4487            .state
4488            .judgements
4489            .iter()
4490            .filter_map(|j| j.ranking.first().copied())
4491            .collect();
4492        let unanimous_initial = tops.len() > 1 && tops.iter().all(|t| *t == tops[0]);
4493
4494        // A judge whose private vote failed still counted once, in the initial
4495        // ranking; using it beats discarding a whole seat.
4496        let mut first_choice: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4497        let mut cast: Vec<char> = Vec::new();
4498        for (i, j) in self.state.judgements.iter().enumerate() {
4499            let vote = self
4500                .state
4501                .votes
4502                .iter()
4503                .find(|v| v.judge == i + 1)
4504                .and_then(|v| v.vote)
4505                .or_else(|| j.ranking.first().copied());
4506            if let Some(v) = vote {
4507                *first_choice.entry(v).or_insert(0) += 1;
4508                cast.push(v);
4509            }
4510        }
4511
4512        let mut borda: BTreeMap<char, usize> = viable.iter().map(|l| (*l, 0)).collect();
4513        for j in &self.state.judgements {
4514            let n = j.ranking.len();
4515            for (pos, label) in j.ranking.iter().enumerate() {
4516                *borda.entry(*label).or_insert(0) += n.saturating_sub(pos + 1);
4517            }
4518        }
4519
4520        let best = first_choice.values().copied().max().unwrap_or(0);
4521        let mut leaders: Vec<char> = first_choice
4522            .iter()
4523            .filter(|(_, v)| **v == best)
4524            .map(|(k, _)| *k)
4525            .collect();
4526        let mut tie_break = None;
4527        if leaders.len() > 1 {
4528            let top_borda = leaders.iter().map(|l| borda[l]).max().unwrap_or(0);
4529            let borda_leaders: Vec<char> = leaders
4530                .iter()
4531                .copied()
4532                .filter(|l| borda[l] == top_borda)
4533                .collect();
4534            tie_break = Some(if borda_leaders.len() == 1 {
4535                format!(
4536                    "{} way tie on first-choice votes, broken by Borda points from the initial rankings",
4537                    leaders.len()
4538                )
4539            } else {
4540                format!(
4541                    "{} way tie on both first-choice votes and Borda points, broken by label order",
4542                    leaders.len()
4543                )
4544            });
4545            leaders = borda_leaders;
4546            leaders.sort_unstable();
4547        }
4548        let winner = *leaders
4549            .first()
4550            .or(viable.first())
4551            .context("no candidate to declare a winner from")?;
4552
4553        let changed_votes = self.state.votes.iter().filter(|v| v.changed).count();
4554        let unanimous_final = !cast.is_empty() && cast.iter().all(|c| *c == cast[0]);
4555        let deliberated = !self.state.deliberation.is_empty();
4556
4557        // Whose verdict is this? A rate-limited seat is absent even if it
4558        // ranked before the limit hit, so presence is measured against the
4559        // recorded losses, not just "did a ranking ever appear".
4560        let quota_seats: std::collections::BTreeSet<&str> =
4561            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4562        let mut present = 0usize;
4563        for (i, j) in self.state.judgements.iter().enumerate() {
4564            if quota_seats.contains(j.seat.as_str()) {
4565                continue;
4566            }
4567            let ranked = !j.ranking.is_empty() && j.failed.is_none();
4568            let voted = self
4569                .state
4570                .votes
4571                .iter()
4572                .any(|v| v.judge == i + 1 && v.vote.is_some());
4573            if ranked || voted {
4574                present += 1;
4575            }
4576        }
4577        // Strict majority of the configured panel. A bare majority is real
4578        // signal we can act on, while a minority verdict must never stand in
4579        // for a healthy one. A one-candidate run needs no panel at all, and
4580        // `judges` stays `0` rather than the roster size a panel that never
4581        // sat would otherwise be credited with.
4582        let needs_quorum = viable.len() > 1;
4583        let judges_total = if needs_quorum {
4584            self.roles.judges.len()
4585        } else {
4586            0
4587        };
4588        let quorum = if needs_quorum {
4589            judges_total / 2 + 1
4590        } else {
4591            0
4592        };
4593        let met_quorum = !needs_quorum || present >= quorum;
4594        let uncontested = (!needs_quorum).then(|| {
4595            format!("only one candidate ({winner}) produced a usable change; no panel was asked")
4596        });
4597
4598        self.state.event(
4599            "tally",
4600            match &uncontested {
4601                Some(reason) => format!("winner {winner} — {reason}"),
4602                None => format!(
4603                    "winner {winner} — votes {} | initial {} | {} changed | \
4604                     {present}/{judges_total} judges{}",
4605                    first_choice
4606                        .iter()
4607                        .map(|(k, v)| format!("{k}:{v}"))
4608                        .collect::<Vec<_>>()
4609                        .join(" "),
4610                    if unanimous_initial {
4611                        "unanimous"
4612                    } else {
4613                        "split"
4614                    },
4615                    changed_votes,
4616                    if met_quorum {
4617                        String::new()
4618                    } else {
4619                        format!(" — below quorum ({quorum} required)")
4620                    },
4621                ),
4622            },
4623        );
4624        if !met_quorum {
4625            self.state.event(
4626                "stall",
4627                format!(
4628                    "verdict rests on {present} of {judges_total} judges (quorum {quorum}); \
4629                     the run stops here, resumable"
4630                ),
4631            );
4632        }
4633        self.state.tally = Some(Tally {
4634            first_choice,
4635            borda,
4636            winner,
4637            rankings: tops.len(),
4638            unanimous_initial,
4639            deliberated,
4640            changed_votes,
4641            unanimous_final,
4642            tie_break,
4643            judges: judges_total,
4644            present,
4645            quorum,
4646            met_quorum,
4647            uncontested,
4648        });
4649        self.state.status = if met_quorum {
4650            RunStatus::Reviewing
4651        } else {
4652            RunStatus::Stalled
4653        };
4654        self.state.save()?;
4655        Ok(())
4656    }
4657
4658    // ------------------------------------------------------------- recover
4659
4660    /// Re-ask the judge seats `tally` counts as absent, so a `Stalled` run can be
4661    /// resumed toward completion once the transient cause clears.
4662    ///
4663    /// A seat is absent — and therefore re-asked — when `tally` refuses to count
4664    /// it toward the quorum, which is exactly the set of seats whose absence
4665    /// collapsed the panel: struck by a rate limit at *any* node (the quorum must
4666    /// not depend on which node happened to hit the limit), or an ordinary
4667    /// failure (`failed = Some`) that never produced a usable ranking. A healthy
4668    /// seat is never disturbed.
4669    ///
4670    /// A seat that now answers with a usable ranking is "recovered": its
4671    /// `Judgement` is refreshed, its `QuotaLoss`/`failed` state cleared (so
4672    /// `tally` counts it present again), and its vote re-collected. A seat that
4673    /// still fails keeps its loss and stays absent.
4674    ///
4675    /// Returns `true` when the re-tally restores the quorum (the run may proceed
4676    /// to review/gate/merge), `false` when it is still below quorum (the run
4677    /// stays `Stalled`, still resumable for a later retry).
4678    #[allow(clippy::too_many_lines)]
4679    async fn recover_stall(&mut self) -> Result<bool> {
4680        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
4681        // agent files with `magi task add` name the run that paid for it. The
4682        // prompt overlay is cloned alongside it because the waves borrow it
4683        // while `self` is mutably borrowed by the node's own bookkeeping.
4684        let run_id = self.state.id.clone();
4685        let prompts = self.state.config.prompts.clone();
4686        // Absent seats = quota-lost at any node, or failed outright. Mirroring
4687        // `tally`'s presence test (rather than the old quota-judge/vote filter)
4688        // is what keeps a non-quota collapse — or a quota loss recorded at the
4689        // deliberate node — from being a permanent dead-end on `--resume`.
4690        let quota_seats: BTreeSet<&str> =
4691            self.state.quota.iter().map(|q| q.seat.as_str()).collect();
4692        let absent: Vec<String> = self
4693            .state
4694            .judgements
4695            .iter()
4696            .filter(|j| quota_seats.contains(j.seat.as_str()) || j.failed.is_some())
4697            .map(|j| j.seat.clone())
4698            .collect();
4699        if absent.is_empty() {
4700            return Ok(false);
4701        }
4702        let viable: Vec<Candidate> = self.state.viable().into_iter().cloned().collect();
4703        if viable.len() <= 1 {
4704            return Ok(false);
4705        }
4706        let labels: Vec<char> = viable.iter().map(|c| c.label).collect();
4707        let language = self.state.config.graph.language.clone();
4708        let timeout = Duration::from_secs(self.state.config.graph.timeout_judge);
4709        let sessions = self.state.config.graph.sessions;
4710        let artifacts = agent::artifacts_dir(&self.state.dir());
4711        let root = self.state.worktree_root();
4712        let base_short = short(&self.state.base_commit);
4713        let candidates: Vec<Candidate> = viable.clone();
4714
4715        // Map each absent seat key to its 0-based position in `roles.judges`.
4716        let mut positions: Vec<usize> = absent
4717            .iter()
4718            .filter_map(|k| self.state.judgements.iter().position(|r| &r.seat == k))
4719            .collect();
4720        if positions.is_empty() {
4721            return Ok(false);
4722        }
4723        positions.sort_unstable();
4724        positions.dedup();
4725
4726        // Re-rank the lost seats, one blind prompt each.
4727        let mut judge_jobs = Vec::new();
4728        for &j in &positions {
4729            let order = blind::presentation_order(viable.len(), j, self.state.seed);
4730            let views: Vec<CandidateView> = order.iter().map(|&k| self.view(&viable[k])).collect();
4731            let seat_key = format!("judge-{}", j + 1);
4732            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4733            let seat = self.seat(&seat_key, &spec.id);
4734            judge_jobs.push(SeatJob {
4735                spec,
4736                seat,
4737                prompt: prompt::judge(
4738                    &self.state.instruction,
4739                    &views,
4740                    self.roles.judges.len(),
4741                    &base_short,
4742                    &language,
4743                ),
4744                cwd: root.join(seat_key),
4745                timeout,
4746                allow_write: false,
4747                sessions,
4748                artifacts: artifacts.clone(),
4749                stem: format!("judge-{}-recover", j + 1),
4750                handover: None,
4751            });
4752        }
4753
4754        let labels_for_check = labels.clone();
4755        let mut judge_losses = Vec::new();
4756        let retries = self.state.config.graph.retries;
4757        let cache = self.state.config.cache_dir();
4758        let ctx = WaveCtx {
4759            carry_seats: false,
4760            pauses: Some((&self.pause, &self.interrupt)),
4761            run: &run_id,
4762            node: "judge",
4763            prompts: &prompts,
4764            cache: cache.as_deref(),
4765            round: None,
4766        };
4767        let results = ask_json_wave::<Ranking>(
4768            judge_jobs,
4769            Arc::clone(&self.sem),
4770            retries,
4771            &self.roles.judge_roster,
4772            &ctx,
4773            &mut judge_losses,
4774            &mut self.state,
4775            &move |r: &Ranking| r.validate(&labels_for_check),
4776        )
4777        .await;
4778        if self.state.resource_block.is_some() {
4779            self.stop_for_resource()?;
4780            return Ok(false);
4781        }
4782
4783        // Refresh the judgement of every seat that ranked again.
4784        let mut recovered: BTreeSet<usize> = BTreeSet::new();
4785        for (&j, (seat, res, _attempts)) in positions.iter().zip(results) {
4786            let agent_id = seat.agent.clone();
4787            self.state.seats.insert(seat.key.clone(), seat);
4788            let record = &mut self.state.judgements[j];
4789            match res {
4790                Ok((ranking, out)) => {
4791                    record.agent = agent_id;
4792                    record.ranking = ranking.normalized();
4793                    record.reasons = ranking.reasons;
4794                    record.strengths = ranking.strengths;
4795                    record.confidence = ranking.confidence;
4796                    record.failed = None;
4797                    record.duration_ms = out.duration_ms;
4798                    recovered.insert(j);
4799                    self.state.event(
4800                        "recover",
4801                        format!("judge {} ranked again after the limit", j + 1),
4802                    );
4803                }
4804                Err(e) => {
4805                    self.state
4806                        .event("recover", format!("judge {} still cannot rank: {e}", j + 1));
4807                }
4808            }
4809        }
4810
4811        // Re-ask the votes of the seats that recovered a ranking.
4812        let mut vote_jobs = Vec::new();
4813        let mut vote_pos: Vec<usize> = Vec::new();
4814        for &j in &recovered {
4815            let seat_key = format!("judge-{}", j + 1);
4816            let spec = self.occupant(&seat_key, self.roles.judges[j].clone());
4817            let seat = self.seat(&seat_key, &spec.id);
4818            let full = self.vote_prompt_full(j, &labels, &language, &candidates, &base_short);
4819            let text = if has_context(&spec, &seat, sessions) {
4820                prompt::final_vote(&labels, &language)
4821            } else {
4822                full.clone()
4823            };
4824            vote_jobs.push(SeatJob {
4825                spec,
4826                seat,
4827                prompt: text,
4828                cwd: root.join(seat_key),
4829                timeout,
4830                allow_write: false,
4831                sessions,
4832                artifacts: artifacts.clone(),
4833                stem: format!("vote-judge-{}-recover", j + 1),
4834                handover: Some(full),
4835            });
4836            vote_pos.push(j);
4837        }
4838        let allowed = labels.clone();
4839        let mut vote_losses = Vec::new();
4840        let vote_retries = self.state.config.graph.retries;
4841        let vote_cache = self.state.config.cache_dir();
4842        let ctx = WaveCtx {
4843            carry_seats: false,
4844            pauses: Some((&self.pause, &self.interrupt)),
4845            run: &run_id,
4846            node: "vote",
4847            prompts: &prompts,
4848            cache: vote_cache.as_deref(),
4849            round: None,
4850        };
4851        let votes = ask_json_wave::<FinalVote>(
4852            vote_jobs,
4853            Arc::clone(&self.sem),
4854            vote_retries,
4855            &self.roles.judge_roster,
4856            &ctx,
4857            &mut vote_losses,
4858            &mut self.state,
4859            &move |v: &FinalVote| match v.label() {
4860                Some(c) if allowed.contains(&c) => Ok(()),
4861                other => bail!("vote {other:?} is not one of {allowed:?}"),
4862            },
4863        )
4864        .await;
4865        if self.state.resource_block.is_some() {
4866            self.stop_for_resource()?;
4867            return Ok(false);
4868        }
4869        for (&j, (seat, res, _attempts)) in vote_pos.iter().zip(votes) {
4870            let agent_id = seat.agent.clone();
4871            self.state.seats.insert(seat.key.clone(), seat);
4872            match res {
4873                Ok((v, _)) => {
4874                    if let Some(rec) = self.state.votes.iter_mut().find(|r| r.judge == j + 1) {
4875                        rec.vote = v.label();
4876                        rec.reason = blind::sanitize_prose(&v.reason, &self.state.config.blind);
4877                    } else {
4878                        self.state.votes.push(VoteRecord {
4879                            judge: j + 1,
4880                            agent: agent_id,
4881                            vote: v.label(),
4882                            reason: blind::sanitize_prose(&v.reason, &self.state.config.blind),
4883                            changed: false,
4884                        });
4885                    }
4886                    self.state.event(
4887                        "recover",
4888                        format!("judge {} voted again after the limit", j + 1),
4889                    );
4890                }
4891                Err(e) => {
4892                    self.state
4893                        .event("recover", format!("judge {} still cannot vote: {e}", j + 1));
4894                }
4895            }
4896        }
4897
4898        // A seat that ranked again is present even if its re-vote failed —
4899        // `tally` falls back to the initial ranking's first choice — so clear
4900        // its quota loss. Seats that still fail keep theirs and stay absent.
4901        let recovered_keys: BTreeSet<String> = recovered
4902            .iter()
4903            .map(|&j| format!("judge-{}", j + 1))
4904            .collect();
4905        self.state
4906            .quota
4907            .retain(|q| !recovered_keys.contains(&q.seat));
4908        // A seat that hit the limit again is a fresh loss, not the old one:
4909        // replace the stale entry so the history stays one-per-seat and the
4910        // daemon can tell this attempt's loss from a previous session's.
4911        for loss in judge_losses.into_iter().chain(vote_losses) {
4912            if recovered_keys.contains(&loss.seat) {
4913                continue;
4914            }
4915            self.state.quota.retain(|q| q.seat != loss.seat);
4916            self.state.quota.push(loss);
4917        }
4918
4919        // Recompute the verdict from the refreshed panel.
4920        self.state.tally = None;
4921        self.tally()?;
4922        Ok(self
4923            .state
4924            .tally
4925            .as_ref()
4926            .map(|t| t.met_quorum)
4927            .unwrap_or(false))
4928    }
4929
4930    // ---------------------------------------------------------------- graft
4931
4932    /// Port the strengths the judges attributed to the losing candidates onto
4933    /// the winner, once, between the tally and the review loop.
4934    ///
4935    /// Best-effort by construction: it returns nothing and fails nothing. A
4936    /// skip, a fixer that failed or hit quota, or a round that changed nothing
4937    /// all leave the winner at its previous commit (`GraftRecord::pre`), and
4938    /// the run carries on as if the phase were off. See [`graft`] for the
4939    /// decisions and `AGENTS.md` for the invariants.
4940    ///
4941    /// The one exception is a rollback that could not be confirmed: the
4942    /// rejected commit may still be the winner's tip, so the run is blocked
4943    /// (resumable; the record stays `Running`, which makes a resume retry the
4944    /// restore) instead of reviewing, gating and merging it.
4945    async fn graft(&mut self) -> Result<()> {
4946        let Err(e) = self.graft_inner().await else {
4947            return Ok(());
4948        };
4949        if e.is::<crate::repo_lock::ResourceBlocked>() {
4950            return Err(e);
4951        }
4952        if e.downcast_ref::<GraftRestoreFailed>().is_some() {
4953            self.state.status = RunStatus::Blocked;
4954            self.state.event("graft", format!("blocked: {e:#}"));
4955            self.state.save()?;
4956            return Err(e);
4957        }
4958        // An error after the `Running` record was saved may have left the
4959        // fixer's work in the tree: the record must not stay unsettled
4960        // (a later resume would reset past the review fixes), so confirm the
4961        // restore now, or block.
4962        if let Some(rec) = self.state.graft.clone()
4963            && !rec.outcome.settled()
4964        {
4965            if let Some(winner) = self.state.winner().cloned() {
4966                if let Err(re) = self.graft_restore(&winner.worktree, &rec.pre).await {
4967                    if re.is::<crate::repo_lock::ResourceBlocked>() {
4968                        return Err(re);
4969                    }
4970                    self.state.status = RunStatus::Blocked;
4971                    self.state.event("graft", format!("blocked: {re:#}"));
4972                    self.state.save()?;
4973                    return Err(re);
4974                }
4975            }
4976            if let Some(g) = self.state.graft.as_mut() {
4977                g.outcome = GraftOutcome::RolledBack(format!("{e:#}"));
4978            }
4979        }
4980        self.state.event("graft", format!("skipped: {e:#}"));
4981        if let Err(e) = self.state.save() {
4982            tracing::warn!("save after graft: {e:#}");
4983        }
4984        Ok(())
4985    }
4986
4987    /// Put the winner back on `pre` and confirm it: a failed `reset` (a stale
4988    /// `index.lock`, say) or a tip that is not `pre` afterwards is an error.
4989    async fn graft_restore(&mut self, wt: &Path, pre: &str) -> Result<()> {
4990        let _coordination = self.coordinate(wt).await?;
4991        crate::recovery::restore_repair(wt, pre)
4992            .await
4993            .map_err(|e| GraftRestoreFailed(format!("{e:#}")).into())
4994    }
4995
4996    /// A skip is a run event, recorded once however often the node is re-entered.
4997    fn graft_skipped(&mut self, why: &str) {
4998        let message = format!("skipped: {why}");
4999        if !self
5000            .state
5001            .events
5002            .iter()
5003            .any(|e| e.node == "graft" && e.message == message)
5004        {
5005            self.state.event("graft", message);
5006        }
5007    }
5008
5009    async fn graft_inner(&mut self) -> Result<()> {
5010        // A round that was started and never finished (crash, kill): the
5011        // fixer may have left anything in the tree, so put the winner back.
5012        if let Some(rec) = self.state.graft.clone()
5013            && !rec.outcome.settled()
5014        {
5015            // Reviews exist only if the graft ended without its record being
5016            // settled (a swallowed error): the tree has moved on since, and
5017            // a reset to `pre` would discard the review fixes.
5018            if self.state.reviews.is_empty()
5019                && let Some(winner) = self.state.winner().cloned()
5020            {
5021                self.graft_restore(&winner.worktree, &rec.pre).await?;
5022            }
5023            if rec.outcome == GraftOutcome::WaitingForRestore {
5024                self.state.graft = None;
5025                self.state.save()?;
5026            } else {
5027                if let Some(g) = self.state.graft.as_mut() {
5028                    g.outcome = GraftOutcome::RolledBack(graft::INTERRUPTED.to_owned());
5029                }
5030                self.state.event(
5031                    "graft",
5032                    "rolled back: the previous round was interrupted; the winner is untouched",
5033                );
5034                self.state.save()?;
5035                return Ok(());
5036            }
5037        }
5038        let plan = match graft::plan(&self.state) {
5039            Ok(plan) => plan,
5040            Err(skip) => {
5041                if skip != graft::Skip::AlreadyDone {
5042                    self.graft_skipped(skip.reason());
5043                    self.state.save()?;
5044                }
5045                return Ok(());
5046            }
5047        };
5048        let Some(winner) = self.state.winner().cloned() else {
5049            return Ok(());
5050        };
5051        let wt = winner.worktree.clone();
5052        if !git::is_clean(&wt).await? {
5053            self.graft_skipped("the winner's worktree has uncommitted changes");
5054            self.state.save()?;
5055            return Ok(());
5056        }
5057        let pre = git::rev_parse(&wt, "HEAD").await?;
5058        // Saved before the fixer is called: a stop from here on is repaired
5059        // by the branch above on the next entry.
5060        self.state.graft = Some(GraftRecord {
5061            pre: pre.clone(),
5062            post: None,
5063            lines: None,
5064            limit: None,
5065            agent: None,
5066            outcome: GraftOutcome::Running,
5067            requested: graft::requested(&plan),
5068            adoption: None,
5069            adoption_error: None,
5070        });
5071        self.state.event(
5072            "graft",
5073            format!(
5074                "porting strengths of candidate(s) {} onto {}",
5075                plan.strengths
5076                    .iter()
5077                    .map(|(l, _, _)| l.to_string())
5078                    .collect::<Vec<_>>()
5079                    .join(", "),
5080                plan.winner
5081            ),
5082        );
5083        self.state.save()?;
5084
5085        let fix_prompt = prompt::graft(
5086            &self.state.instruction,
5087            plan.winner,
5088            &plan.strengths,
5089            &self.state.config.graph.language,
5090        );
5091        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
5092        let sessions = self.state.config.graph.sessions;
5093        let artifacts = agent::artifacts_dir(&self.state.dir());
5094        let attempts = fixer::graft_attempts(&self.state, &self.roles, &winner);
5095        let (_, seat, out) = self
5096            .ask_fixer_among(attempts, "graft", None, Some((&wt, &pre)), |spec, seat| {
5097                SeatJob {
5098                    prompt: fix_prompt.clone(),
5099                    spec,
5100                    seat,
5101                    cwd: wt.clone(),
5102                    timeout,
5103                    allow_write: true,
5104                    sessions,
5105                    artifacts: artifacts.clone(),
5106                    stem: "graft".to_owned(),
5107                    handover: None,
5108                }
5109            })
5110            .await;
5111        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
5112            // Preserve the need to restore earlier fallback edits without
5113            // confusing an unstarted seat with an interrupted invocation.
5114            self.state.graft.as_mut().expect("graft record").outcome =
5115                GraftOutcome::WaitingForRestore;
5116            return self.stop_for_resource();
5117        }
5118        let mut why: Option<String> = None;
5119        match &out {
5120            AgentOutcome::Ok(o) if agent::output_advances(o) => {
5121                why = Some("the fixer produced nothing usable".to_owned());
5122            }
5123            AgentOutcome::Ok(_) => {}
5124            AgentOutcome::Dropped(_) => why = Some("the CLI dropped the stream".to_owned()),
5125            AgentOutcome::Quota(o) => {
5126                self.state.quota.push(QuotaLoss::from_report(
5127                    seat.key.clone(),
5128                    "graft",
5129                    o.quota.as_ref(),
5130                ));
5131                why = Some("rate limited (quota)".to_owned());
5132            }
5133            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
5134                why = Some(format!(
5135                    "the fixer failed: {}",
5136                    e.lines()
5137                        .next()
5138                        .unwrap_or("")
5139                        .chars()
5140                        .take(160)
5141                        .collect::<String>()
5142                ));
5143            }
5144        }
5145        let agent_id = seat.agent.clone();
5146        self.state.seats.insert(seat.key.clone(), seat);
5147
5148        let mut post = None;
5149        let mut lines = None;
5150        let _coordination = self.coordinate(&wt).await?;
5151        if why.is_none() {
5152            let rescued = git::rescue_commit(&wt, "magi: graft (uncommitted work)").await;
5153            if let Ok(r) = &rescued {
5154                self.state.note_withheld("graft", &r.withheld);
5155            }
5156            let head = git::rev_parse(&wt, "HEAD").await?;
5157            // What is measured must be what gets reviewed and gated: a failed
5158            // rescue, or anything still uncommitted (withheld lockfiles
5159            // aside, which are never part of the tree), rejects the graft.
5160            let dirty = match &rescued {
5161                Ok(r) => git::status_porcelain(&wt)
5162                    .await
5163                    .map(|st| {
5164                        st.lines().any(|l| {
5165                            !r.withheld
5166                                .iter()
5167                                .any(|w| l.trim_end().ends_with(w.path.as_str()))
5168                        })
5169                    })
5170                    .unwrap_or(true),
5171                Err(_) => true,
5172            };
5173            if dirty {
5174                why = Some("the fixer left changes that could not be committed".to_owned());
5175            } else if head == pre {
5176                why = Some("the fixer changed nothing".to_owned());
5177            } else {
5178                lines = git::changed_lines(&wt, &pre, &head).await;
5179                post = Some(head);
5180            }
5181        }
5182        let (adoption, adoption_error) = match &out {
5183            AgentOutcome::Ok(o) => crate::graft_report::adoption_of(&o.text),
5184            _ => (None, None),
5185        };
5186        let requested = graft::requested(&plan);
5187        let outcome = match why {
5188            None => {
5189                self.state.event(
5190                    "graft",
5191                    format!(
5192                        "applied: {}",
5193                        lines.map_or_else(
5194                            || "changed lines unavailable".to_owned(),
5195                            |n| format!("{n} changed lines")
5196                        )
5197                    ),
5198                );
5199                GraftOutcome::Applied
5200            }
5201            Some(why) => {
5202                // Not settled until the restore is confirmed: the record
5203                // stays `Running`, so a resume retries it.
5204                crate::recovery::restore_repair(&wt, &pre)
5205                    .await
5206                    .map_err(|e| GraftRestoreFailed(format!("{e:#}")))?;
5207                self.state.event(
5208                    "graft",
5209                    format!("rolled back: {why}; the winner is untouched"),
5210                );
5211                GraftOutcome::RolledBack(why)
5212            }
5213        };
5214        self.state.graft = Some(GraftRecord {
5215            pre,
5216            post,
5217            lines,
5218            limit: None,
5219            agent: Some(agent_id),
5220            outcome,
5221            requested,
5222            adoption,
5223            adoption_error,
5224        });
5225        self.state.save()?;
5226        Ok(())
5227    }
5228
5229    // ----------------------------------------------------------------- fold
5230
5231    async fn fold_losers(&mut self) -> Result<()> {
5232        let Some(winner) = self.state.tally.as_ref().map(|t| t.winner) else {
5233            return Ok(());
5234        };
5235        if !self.state.candidate_preservation.is_empty() {
5236            bail!("candidate preservation incomplete; refusing cleanup");
5237        }
5238        let repo = self.state.repo.clone();
5239        let _coordination = self.coordinate(&repo).await?;
5240        let mut folded = Vec::new();
5241        for i in 0..self.state.candidates.len() {
5242            let c = &self.state.candidates[i];
5243            if c.label == winner || c.folded {
5244                continue;
5245            }
5246            let (wt, branch, label) = (c.worktree.clone(), c.branch.clone(), c.label);
5247            remove_preserved_worktree(&repo, &wt).await?;
5248            if git::branch_exists(&repo, &branch).await?
5249                && !git::branch_delete(&repo, &branch).await?
5250            {
5251                bail!("candidate branch removal refused");
5252            }
5253            self.state.candidates[i].folded = true;
5254            self.state.save()?;
5255            folded.push(label.to_string());
5256        }
5257        // The judges are finished; their checkouts are pure cost from here.
5258        let root = self.state.worktree_root();
5259        for j in 1..=self.roles.judges.len() {
5260            let wt = root.join(format!("judge-{j}"));
5261            if wt.exists() {
5262                remove_preserved_worktree(&repo, &wt).await?;
5263            }
5264        }
5265        // The design-deliberation stage is finished by the time a tally
5266        // exists — same reasoning as the judges above.
5267        if self.state.config.graph.advise {
5268            for k in 1..=self.state.config.graph.advisors {
5269                let wt = root.join(format!("advisor-{k}"));
5270                if wt.exists() {
5271                    remove_preserved_worktree(&repo, &wt).await?;
5272                }
5273            }
5274        }
5275        if !folded.is_empty() {
5276            self.state
5277                .event("fold", format!("folded candidates {}", folded.join(", ")));
5278            self.state.save()?;
5279        }
5280        Ok(())
5281    }
5282
5283    // ------------------------------------------------------------ base sync
5284
5285    async fn hold_base_sync(&mut self) -> Result<bool> {
5286        self.hold_base_sync_as(None).await
5287    }
5288
5289    async fn hold_base_sync_kind(
5290        &mut self,
5291        kind: crate::base_sync_recovery::StopKind,
5292    ) -> Result<()> {
5293        self.hold_base_sync_as(Some(kind)).await?;
5294        Ok(())
5295    }
5296
5297    async fn hold_base_sync_as(
5298        &mut self,
5299        kind: Option<crate::base_sync_recovery::StopKind>,
5300    ) -> Result<bool> {
5301        if !self.state.base_sync_blocked() {
5302            self.state.base_sync_recovery.stop = None;
5303            return Ok(false);
5304        }
5305        let head = match self.state.winner() {
5306            Some(w) => git::rev_parse(&w.worktree, "HEAD")
5307                .await
5308                .unwrap_or_default(),
5309            None => String::new(),
5310        };
5311        if let Some(authorization) = self.state.base_sync_recovery.authorizations.last_mut()
5312            && !authorization.consumed
5313            && authorization.stop.head != head
5314        {
5315            authorization.refuse(format!(
5316                "retained HEAD changed from {} to {head}; inspect it and authorize again with --recover-base",
5317                authorization.stop.head
5318            ));
5319        }
5320        self.state.record_base_sync_stop(&head);
5321        if let Some(kind) = kind {
5322            self.state.base_sync_recovery.stop.as_mut().unwrap().kind = kind.clone();
5323            if let Some(stop) = self.state.base_sync_recovery.stops.last_mut() {
5324                stop.kind = kind;
5325            }
5326        }
5327        self.state.save()?;
5328        Ok(true)
5329    }
5330
5331    /// Land the winner's tree on the current tip of `<remote>/<base>` before
5332    /// anything verifies it.
5333    ///
5334    /// `verify.e2e`, `verify.gate` and every reviewer in [`Self::review_loop`]
5335    /// read whatever is checked out in the winner's worktree. Left alone that
5336    /// tree stays rooted at `base_commit` - the base as [`resolve_base`] saw
5337    /// it when the run *branched* - and a run takes long enough that the base
5338    /// has usually moved by the time it gets here. A gate that ran there
5339    /// answers "green on the commit this run started from", not "green on
5340    /// what is about to land", and the difference showed up three times in
5341    /// one day as a green run whose merge would have reverted a file another
5342    /// pull request had already landed.
5343    ///
5344    /// Reuses [`crate::rebase::rebase_with_fixer`], the same routine
5345    /// `land::Step::Rebase` calls, rather than a second implementation of the
5346    /// same idea: a throwaway worktree, nothing runs in the primary tree, and
5347    /// a second rebase path is exactly the kind of drift `resolve_base`'s own
5348    /// doc warns about ("two answers to a question nobody notices until a
5349    /// diff is wrong").
5350    ///
5351    /// Foreign-checkout reviews instead merge the base into the owned shadow
5352    /// worktree, preserving the ancestry required for fast-forward delivery.
5353    /// A conflicting merge is aborted and held for operator resolution.
5354    ///
5355    /// A conflict is not the end of the road: the standing rebase is handed
5356    /// to the fixer seat, at most `graph.review_rounds` times, counted in
5357    /// `state.rebase_fixes` (so it survives a park/resume and is shared with
5358    /// land). Once it finishes, review and the gate run as usual on the
5359    /// rebased tree, which is where a breakage the new base caused is caught
5360    /// by the ordinary gate-fix round. magi resolves nothing itself.
5361    ///
5362    /// Two different bounds, easy to confuse: [`BASE_SYNC_ROUNDS`], counted in
5363    /// the current durable recovery interval, bounds how often the base is
5364    /// synchronized during active recovery (lifetime attempts remain evidence).
5365    /// `rebase_fixes` is how many times a *conflict* was given to a fixer. When the fixer cannot finish the
5366    /// rebase the branch is restored, `state.base_sync.conflict` is set with
5367    /// what was tried (rounds spent, paths still conflicted) and the branch
5368    /// and worktree stay exactly as they were - untouched, for a person to
5369    /// look at - which is also what makes re-entering this function
5370    /// afterwards a no-op instead of a second attempt at the same wall. A
5371    /// push failure ends the same way.
5372    async fn sync_to_base(&mut self) -> Result<()> {
5373        if self.state.status == RunStatus::AlreadyInBase {
5374            return Ok(());
5375        }
5376        self.hold_base_sync().await?;
5377        let conflicted = self
5378            .state
5379            .base_sync
5380            .as_ref()
5381            .is_some_and(|s| s.conflict.is_some());
5382        let Some(winner) = self.state.winner().cloned() else {
5383            return Ok(());
5384        };
5385
5386        crate::worktree_clean::resume_pending(&mut self.state, &winner.worktree).await?;
5387
5388        let repo = self.state.repo.clone();
5389        let remote = self.state.config.merge.remote.clone();
5390        let base_branch = self.state.base_branch.clone();
5391        let tracking = format!("{remote}/{base_branch}");
5392
5393        let fetched_base =
5394            matches!(git::fetch(&repo, &remote, &base_branch).await, Ok(o) if o.ok());
5395        // No network, or the remote never had this branch: the run already
5396        // started from a fetched `<remote>/<base>` (`resolve_base` refuses
5397        // otherwise), and one that got this far is not blocked by a fetch
5398        // that fails now. It just has no newer tip to compare against.
5399        let Ok(mut tip) = git::rev_parse(&repo, &tracking).await else {
5400            return Ok(());
5401        };
5402
5403        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
5404        self.state.ensure_base_episode();
5405        let mut pending = self
5406            .state
5407            .base_sync_recovery
5408            .operations
5409            .last()
5410            .filter(|o| !o.finished)
5411            .cloned();
5412        let scratch = self.state.dir().join("base-sync");
5413        if git::rebase_in_progress(&scratch).await {
5414            // Older resource stops predate operation reservations. Import the
5415            // standing Git operation, never retarget it to today's main.
5416            if pending.is_none()
5417                && self.state.base_sync_recovery.operations.is_empty()
5418                && !conflicted
5419                && crate::rebase::saved_base_identity(&scratch, &winner.branch)
5420                    .await
5421                    .ok()
5422                    .as_deref()
5423                    == Some(head.as_str())
5424                && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5425                && git::is_clean(&winner.worktree).await?
5426            {
5427                let saved_tip = crate::rebase::rebase_target(&scratch).await?;
5428                if self.state.base_sync.is_none() {
5429                    self.state.base_sync = Some(BaseSync {
5430                        tip: saved_tip.clone(),
5431                        behind: 0,
5432                        attempts: 0,
5433                        conflict: None,
5434                        already_in: None,
5435                    });
5436                }
5437                self.state.reserve_base_operation(&head, &saved_tip);
5438                self.state.event(
5439                    "base-recovery",
5440                    "imported standing legacy rebase as a spent synchronization reservation",
5441                );
5442                self.state.save()?;
5443                pending = self.state.base_sync_recovery.operations.last().cloned();
5444            }
5445            let expected = pending.as_ref().map(|op| op.tip.as_str());
5446            if expected.is_none()
5447                || crate::rebase::rebase_target(&scratch).await.as_deref().ok() != expected
5448            {
5449                let why = "saved scratch rebase targets a different base; inspect the retained rebase before resuming";
5450                if self.state.base_sync.is_none() {
5451                    self.state.base_sync = Some(BaseSync {
5452                        tip: tip.clone(),
5453                        behind: 0,
5454                        attempts: 0,
5455                        conflict: None,
5456                        already_in: None,
5457                    });
5458                }
5459                self.state.base_sync.as_mut().unwrap().conflict = Some(why.into());
5460                self.state.event("land", why);
5461                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5462                    .await?;
5463                return Ok(());
5464            }
5465        }
5466        if let Some(op) = &pending {
5467            // A replay may add the pinned base's commits, never foreign work.
5468            let mut replay_proven =
5469                head == op.head || op.validated_head.as_deref() == Some(head.as_str());
5470            if !replay_proven && scratch.exists() && !git::rebase_in_progress(&scratch).await {
5471                let wait = repository_wait(&self.state, None);
5472                let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5473                match crate::rebase::recover_completed_base(&self.state, &scratch, &winner.branch, op, &head).await {
5474                    Ok(true) => {
5475                        self.state.base_sync_recovery.operations.last_mut().unwrap().validated_head = Some(head.clone());
5476                        self.state.save()?;
5477                        git::worktree_remove(&repo, &scratch).await.ok();
5478                        replay_proven = true;
5479                    }
5480                    Ok(false) => self.state.event("base-recovery", "completed scratch failed result validation; retained for inspection"),
5481                    Err(error) => self.state.event("base-recovery", format!("completed scratch could not be validated; retained for inspection: {error:#}")),
5482                }
5483            }
5484            if !replay_proven && git::is_ancestor(&repo, &op.tip, &head).await {
5485                if self.state.foreign_checkout.is_some() {
5486                    replay_proven = git::rev_parse(&repo, &format!("{head}^1"))
5487                        .await
5488                        .ok()
5489                        .as_deref()
5490                        == Some(op.head.as_str())
5491                        && git::rev_parse(&repo, &format!("{head}^2"))
5492                            .await
5493                            .ok()
5494                            .as_deref()
5495                            == Some(op.tip.as_str());
5496                } else if crate::reconcile::origin_missing(&repo, &head, &op.head)
5497                    .await
5498                    .is_ok_and(|m| m.is_empty())
5499                    && let Ok(added) =
5500                        crate::reconcile::origin_missing(&repo, &op.head, &head).await
5501                {
5502                    replay_proven = true;
5503                    for commit in added {
5504                        if !git::is_ancestor(&repo, &commit, &op.tip).await {
5505                            replay_proven = false;
5506                            break;
5507                        }
5508                    }
5509                    replay_proven &= git::commits_ahead(&repo, &op.tip, &head).await?
5510                        <= git::commits_ahead(&repo, &op.tip, &op.head).await?;
5511                }
5512            }
5513            // Never retarget an operation across a crash, even if main moved.
5514            // sync_to_head resets and cleans with -x, so even ignored untracked
5515            // files must stop recovery independently of the old tracked tree.
5516            if git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5517                || (head != op.head
5518                    && !git::git(&winner.worktree, &["ls-files", "--others", "-z"])
5519                        .await?
5520                        .is_empty())
5521                || (!git::is_clean(&winner.worktree).await?
5522                    && !git::git_raw(&winner.worktree, &["diff", "--quiet", &op.head])
5523                        .await
5524                        .is_ok_and(|o| o.ok()))
5525                || !replay_proven
5526            {
5527                let why = format!(
5528                    "unfinished base synchronization {} has unrecognized branch/worktree state; inspect the retained branch and scratch rebase before resuming",
5529                    op.id
5530                );
5531                let sync = self
5532                    .state
5533                    .base_sync
5534                    .as_mut()
5535                    .context("reserved synchronization has no lifetime accounting")?;
5536                sync.conflict = Some(why.clone());
5537                self.state.event("land", why);
5538                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Unknown)
5539                    .await?;
5540                return Ok(());
5541            }
5542            if head != op.head {
5543                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5544            }
5545            tip = op.tip.clone();
5546        }
5547        let behind = git::commits_ahead(&repo, &head, &tip).await.unwrap_or(0);
5548        let attempts = self.state.base_sync.as_ref().map_or(0, |s| s.attempts);
5549        if pending.is_none()
5550            && !conflicted
5551            && fetched_base
5552            && self
5553                .state
5554                .base_sync_recovery
5555                .episodes
5556                .last()
5557                .is_some_and(|e| e.boundary.is_some())
5558            && let Some(task) = self.state.origin.as_ref().and_then(|o| o.task.clone())
5559            && let Ok(owner) = crate::queue::Queue::open().get(&task)
5560            && owner.execution_runs().last() == Some(&self.state.id)
5561            && owner.status != crate::queue::TaskStatus::Done
5562            && !owner.fresh_start
5563            && owner.review_branch.is_none()
5564            && crate::daemon::compatible_run_instruction(&self.state, &owner)
5565            && let Some(approval) = owner.resumption.as_ref()
5566            && approval.instruction == owner.instruction
5567            && git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? == head
5568            && git::is_clean(&winner.worktree).await?
5569            && self.state.renew_base_episode(&task, approval, &head, &tip)
5570        {
5571            self.state.event("base-recovery", "opened bounded synchronization interval after owner-approved infrastructure recovery");
5572            self.state.save()?;
5573        }
5574
5575        // Before any rebase, and before a recorded conflict is honoured: a
5576        // branch whose change reached the base under other commit ids has
5577        // nothing to rebase and nothing to conflict with, and a run that
5578        // already stopped on that phantom conflict recovers here on resume.
5579        // `behind == 0` with head == tip is a branch the base has since taken
5580        // in whole, whether or not a conflict was ever recorded: the ancestry
5581        // proof must still run (`classify` ignores a head still on the start
5582        // commit).
5583        if (behind > 0 || conflicted || head == tip)
5584            && self
5585                .settle_already_in(&winner.branch, &tip, &head, attempts, behind)
5586                .await?
5587        {
5588            return Ok(());
5589        }
5590        if conflicted && self.state.base_recovery_pending() {
5591            let authorization = self.state.base_sync_recovery.authorizations.last().unwrap();
5592            let owner =
5593                crate::queue::Queue::at(crate::run::home().join("queue")).get(&authorization.task);
5594            let owner_valid = owner.as_ref().is_ok_and(|t| {
5595                t.execution_runs().last() == Some(&self.state.id)
5596                    && t.status != crate::queue::TaskStatus::Done
5597                    && !t.fresh_start
5598                    && t.review_branch.is_none()
5599            });
5600            let invalidation = if !owner_valid {
5601                Some("latest task ownership changed")
5602            } else if !owner.as_ref().is_ok_and(|t| {
5603                authorization
5604                    .generation
5605                    .is_none_or(|g| t.resumption.as_ref().is_some_and(|r| r.generation == g))
5606            }) {
5607                Some("owner approval generation changed")
5608            } else if let Some(scope) = &authorization.scope {
5609                if scope.tip != tip {
5610                    Some("fetched base changed")
5611                } else if !self
5612                    .state
5613                    .base_sync_recovery
5614                    .episodes
5615                    .last()
5616                    .is_some_and(|e| e.id == scope.episode)
5617                {
5618                    Some("active synchronization interval changed")
5619                } else if !owner
5620                    .as_ref()
5621                    .is_ok_and(|t| t.instruction == scope.instruction)
5622                {
5623                    Some("task instruction changed")
5624                } else {
5625                    None
5626                }
5627            } else {
5628                None
5629            };
5630            if let Some(reason) = invalidation {
5631                let authorization = self
5632                    .state
5633                    .base_sync_recovery
5634                    .authorizations
5635                    .last_mut()
5636                    .unwrap();
5637                authorization.refuse(format!("{reason}; authorize again with --recover-base"));
5638                self.state.save()?;
5639                return Ok(());
5640            }
5641            if !fetched_base
5642                || git::rev_parse(&repo, &format!("refs/heads/{}", winner.branch)).await? != head
5643                || !git::is_clean(&winner.worktree).await?
5644                || !owner.is_ok_and(|t| t.execution_runs().last() == Some(&self.state.id))
5645            {
5646                return Ok(());
5647            }
5648        }
5649        let recovery = conflicted && self.state.consume_base_recovery(&head);
5650        if conflicted {
5651            // Consumption (including a refused changed HEAD) is durable before Git.
5652            self.state.save()?;
5653            if !recovery {
5654                return Ok(());
5655            }
5656        }
5657
5658        if behind == 0 {
5659            // A fixer-finished rebase moves the branch ref before the
5660            // winner's worktree is told (`sync_to_head` below). A run that
5661            // died in between resumes here with `behind == 0` and a tree still
5662            // holding the pre-rebase files, which review and the gate would
5663            // then read. That state is exactly: HEAD moved off the tip the
5664            // rebase started from, yet the tree is still identical to that
5665            // tip. A tree with edits of its own differs from it, so nothing
5666            // is thrown away.
5667            if let Some(from) = self
5668                .state
5669                .rebase_fixes
5670                .iter()
5671                .rev()
5672                .find_map(|r| r.from.clone())
5673                && from != head
5674                && git::git_raw(&winner.worktree, &["diff", "--quiet", &from])
5675                    .await
5676                    .is_ok_and(|o| o.ok())
5677            {
5678                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5679            }
5680            // An earlier attempt may have rebased the branch locally and died
5681            // before pushing it (only the fresh-rebase arm below pushes).
5682            // Publish it now, so the plain push at PR time is not refused as
5683            // a non-fast-forward. A run already holding a recorded conflict
5684            // never reaches here; that case is out of scope.
5685            let conflict = self.publish_resumed_rebase(&winner.branch, &head).await;
5686            if let Some(why) = &conflict {
5687                self.state.status = RunStatus::Blocked;
5688                self.state.event("land", why.clone());
5689            }
5690            self.state.base_sync = Some(BaseSync {
5691                tip,
5692                behind: 0,
5693                attempts,
5694                conflict,
5695                already_in: None,
5696            });
5697            if pending.is_some() {
5698                self.state.finish_base_operation(Some(&head));
5699            }
5700            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Publication)
5701                .await?;
5702            self.state.save()?;
5703            return Ok(());
5704        }
5705
5706        if self.state.base_episode_used() >= BASE_SYNC_ROUNDS && pending.is_none() && !recovery {
5707            let why = format!(
5708                "{base_branch} moved {behind} commit(s) ahead of {} after {BASE_SYNC_ROUNDS} \
5709                 synchronization(s); synchronizing again would only race it",
5710                winner.branch
5711            );
5712            self.state.status = RunStatus::Blocked;
5713            self.state.base_sync = Some(BaseSync {
5714                tip,
5715                behind,
5716                attempts,
5717                conflict: Some(why.clone()),
5718                already_in: None,
5719            });
5720            self.state.event("land", why);
5721            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Exhausted)
5722                .await?;
5723            self.state.save()?;
5724            return Ok(());
5725        }
5726
5727        if self.state.foreign_checkout.is_some() {
5728            // Delivery to the operator's checkout is fast-forward only. A
5729            // rebase here would discard its saved head from our ancestry.
5730            let wait = repository_wait(&self.state, None);
5731            let _coordination = coordinate_operation(&mut self.state, &repo, &wait).await?;
5732            if self.state.base_sync.is_none() {
5733                self.state.base_sync = Some(BaseSync {
5734                    tip: tip.clone(),
5735                    behind,
5736                    attempts: 0,
5737                    conflict: None,
5738                    already_in: None,
5739                });
5740            }
5741            self.state.reserve_base_operation(&head, &tip);
5742            self.state.save()?;
5743            let merged = git::git_raw(
5744                &winner.worktree,
5745                &[
5746                    "-c",
5747                    "core.hooksPath=/dev/null",
5748                    "merge",
5749                    "--no-ff",
5750                    "--no-autostash",
5751                    "--no-edit",
5752                    "-m",
5753                    &format!("Merge {tracking} before verifying"),
5754                    &tip,
5755                ],
5756            )
5757            .await?;
5758            let conflict = if merged.ok() {
5759                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5760                self.state.event(
5761                    "land",
5762                    format!("merged {tracking} into {} before verifying", winner.branch),
5763                );
5764                None
5765            } else {
5766                if git::rev_exists(&winner.worktree, "MERGE_HEAD").await {
5767                    git::git(&winner.worktree, &["merge", "--abort"]).await?;
5768                }
5769                let why = format!(
5770                    "could not merge {tracking} into {} without rewriting foreign checkout history: {}",
5771                    winner.branch,
5772                    merged.stderr.chars().take(600).collect::<String>()
5773                );
5774                self.state.status = RunStatus::Blocked;
5775                self.state.event("land", &why);
5776                Some(why)
5777            };
5778            let synced = conflict.is_none();
5779            let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5780            self.state
5781                .finish_base_operation(synced.then_some(fresh.as_str()));
5782            let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5783            self.state.base_sync = Some(BaseSync {
5784                tip,
5785                behind: if synced { 0 } else { behind },
5786                attempts,
5787                conflict,
5788                already_in: None,
5789            });
5790            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Conflict)
5791                .await?;
5792            self.state.save()?;
5793            return Ok(());
5794        }
5795
5796        self.state.event(
5797            "land",
5798            format!(
5799                "{base_branch} moved {behind} commit(s) ahead of {}; rebasing before verifying",
5800                winner.branch
5801            ),
5802        );
5803        self.state.save()?;
5804
5805        // The remote's copy of the branch, read now and only if the fetch
5806        // really succeeded (a stale tracking ref must never pin a lease). It is
5807        // pushed over after a rebase only when it is a commit this branch
5808        // already contains, by ancestry or by patch (an earlier rebase of ours
5809        // that never reached the remote): anything else is somebody else's work.
5810        let branch_tracking = format!("{remote}/{}", winner.branch);
5811        let fetched_branch = git::fetch(&repo, &remote, &winner.branch).await;
5812        let remote_tip = if self.state.task_pr_branch().is_none()
5813            && matches!(&fetched_branch, Ok(o) if o.ok())
5814        {
5815            git::rev_parse(&repo, &branch_tracking).await.ok()
5816        } else {
5817            None
5818        };
5819        // A remote tip this branch does not contain is somebody else's work:
5820        // rebasing would leave a local tip that can never be pushed. Stop
5821        // before touching anything and say so.
5822        if let Some(theirs) = &remote_tip
5823            && !git::is_ancestor(&repo, theirs, &head).await
5824            && !crate::reconcile::origin_missing(&repo, &head, theirs)
5825                .await
5826                .is_ok_and(|missing| missing.is_empty())
5827        {
5828            let why = format!(
5829                "{branch_tracking} ({}) has commits {} does not contain; not rebasing over \
5830                 them",
5831                short(theirs),
5832                winner.branch
5833            );
5834            self.state.status = RunStatus::Blocked;
5835            self.state.base_sync = Some(BaseSync {
5836                tip,
5837                behind,
5838                attempts,
5839                conflict: Some(why.clone()),
5840                already_in: None,
5841            });
5842            self.state.event("land", why);
5843            self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::Divergence)
5844                .await?;
5845            self.state.save()?;
5846            return Ok(());
5847        }
5848
5849        if self.state.base_sync.is_none() {
5850            self.state.base_sync = Some(BaseSync {
5851                tip: tip.clone(),
5852                behind,
5853                attempts: 0,
5854                conflict: None,
5855                already_in: None,
5856            });
5857        }
5858        self.state.reserve_base_operation(&head, &tip);
5859        self.state.save()?;
5860        let scratch = self.state.dir().join("base-sync");
5861        let rebased =
5862            match crate::rebase::rebase_with_fixer(&mut self.state, &scratch, &winner.branch, &tip)
5863                .await
5864            {
5865                Ok(crate::rebase::Rebased::Applied) => Ok(None),
5866                Ok(crate::rebase::Rebased::Stopped(why)) => Ok(Some(why)),
5867                Err(e) => Err(e),
5868            };
5869        let attempts = self.state.base_sync.as_ref().unwrap().attempts;
5870        let stop_kind = match &rebased {
5871            Ok(None) => crate::base_sync_recovery::StopKind::Publication,
5872            Ok(Some(_)) => crate::base_sync_recovery::StopKind::Conflict,
5873            Err(_) => crate::base_sync_recovery::StopKind::Unknown,
5874        };
5875        match rebased {
5876            Ok(None) => {
5877                // The branch ref moved, but a worktree that already had it
5878                // checked out (the winner's) was not told; sync its index and
5879                // files before anything reads them.
5880                crate::worktree_clean::reset(&mut self.state, &winner.worktree, None).await?;
5881                let fresh = git::rev_parse(&winner.worktree, "HEAD").await?;
5882                self.state.finish_base_operation(Some(&fresh));
5883                refresh_reviewed_commits(&mut self.state, &winner.branch).await;
5884                let mut conflict = None;
5885                if let Some(pinned) = &remote_tip {
5886                    let pushed = git::push_pinned(&repo, &remote, &winner.branch, pinned).await;
5887                    match pushed {
5888                        Ok(o) if o.ok() => self.state.event(
5889                            "land",
5890                            format!("pushed rebased {} to {remote}", winner.branch),
5891                        ),
5892                        Ok(o) => {
5893                            conflict = Some(format!(
5894                                "rebased {} locally but {remote} refused the push (it moved                                  since {}; someone may have pushed): {}",
5895                                winner.branch,
5896                                short(pinned),
5897                                o.stderr.chars().take(600).collect::<String>()
5898                            ));
5899                        }
5900                        Err(e) => {
5901                            conflict = Some(format!(
5902                                "rebased {} locally but could not push it: {e:#}",
5903                                winner.branch
5904                            ));
5905                        }
5906                    }
5907                }
5908                if let Some(why) = &conflict {
5909                    self.state.status = RunStatus::Blocked;
5910                    self.state.event("land", why.clone());
5911                }
5912                self.state.base_sync = Some(BaseSync {
5913                    tip: tip.clone(),
5914                    behind: 0,
5915                    attempts,
5916                    conflict,
5917                    already_in: None,
5918                });
5919                self.state
5920                    .event("land", format!("rebased {} onto {tracking}", winner.branch));
5921            }
5922            Ok(Some(conflict)) => {
5923                self.state.finish_base_operation(None);
5924                let why = format!(
5925                    "{} conflicts with {tracking} and did not rebase: {}",
5926                    winner.branch,
5927                    conflict.chars().take(600).collect::<String>()
5928                );
5929                self.state.status = RunStatus::Blocked;
5930                self.state.base_sync = Some(BaseSync {
5931                    tip,
5932                    behind,
5933                    attempts,
5934                    conflict: Some(why.clone()),
5935                    already_in: None,
5936                });
5937                self.state.event("land", why);
5938            }
5939            // A refused Git write or a busy repository: nothing is wrong with
5940            // the branch. The scratch rebase stays standing and the run
5941            // resumes into it (`base_sync` must not claim a conflict).
5942            Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
5943                self.state.save()?;
5944                return Ok(());
5945            }
5946            Err(e) => {
5947                self.state.finish_base_operation(None);
5948                let why = format!("could not rebase {} onto {tracking}: {e:#}", winner.branch);
5949                self.state.status = RunStatus::Blocked;
5950                self.state.base_sync = Some(BaseSync {
5951                    tip,
5952                    behind,
5953                    attempts,
5954                    conflict: Some(why.clone()),
5955                    already_in: None,
5956                });
5957                self.state.event("land", why);
5958            }
5959        }
5960        if !self.state.base_sync_blocked() {
5961            self.state.base_sync_recovery.stop = None;
5962        }
5963        self.hold_base_sync_kind(stop_kind).await?;
5964        self.state.save()?;
5965        Ok(())
5966    }
5967
5968    /// Push a branch an earlier attempt rebased locally but never published,
5969    /// pinned to the remote tip read right after a successful fetch. Returns
5970    /// the reason when the run must stop; `None` when there was nothing to do
5971    /// (no remote copy, the same tip, or a remote copy this branch already
5972    /// contains, which the PR-time push fast-forwards) or the push succeeded.
5973    async fn publish_resumed_rebase(&mut self, branch: &str, head: &str) -> Option<String> {
5974        // Task competitions publish only after the PR text gate, to their
5975        // public head; never resurrect a candidate branch on the remote.
5976        if self.state.task_pr_branch().is_some() {
5977            return None;
5978        }
5979        let repo = self.state.repo.clone();
5980        let remote = self.state.config.merge.remote.clone();
5981        let fetched = git::fetch(&repo, &remote, branch).await;
5982        if !matches!(&fetched, Ok(o) if o.ok()) {
5983            return None;
5984        }
5985        let branch_tracking = format!("{remote}/{branch}");
5986        let theirs = git::rev_parse(&repo, &branch_tracking).await.ok()?;
5987        if theirs == head || git::is_ancestor(&repo, &theirs, head).await {
5988            return None;
5989        }
5990        if !crate::reconcile::origin_missing(&repo, head, &theirs)
5991            .await
5992            .is_ok_and(|missing| missing.is_empty())
5993        {
5994            return Some(format!(
5995                "{branch_tracking} ({}) has commits {branch} does not contain; not pushing over \
5996                 them",
5997                short(&theirs)
5998            ));
5999        }
6000        match git::push_pinned(&repo, &remote, branch, &theirs).await {
6001            Ok(o) if o.ok() => {
6002                self.state
6003                    .event("land", format!("pushed rebased {branch} to {remote}"));
6004                None
6005            }
6006            Ok(o) => Some(format!(
6007                "{branch} is rebased locally but {remote} refused the push (it moved since {}; \
6008                 someone may have pushed): {}",
6009                short(&theirs),
6010                o.stderr.chars().take(600).collect::<String>()
6011            )),
6012            Err(e) => Some(format!(
6013                "{branch} is rebased locally but could not be pushed: {e:#}"
6014            )),
6015        }
6016    }
6017
6018    /// End the run as [`RunStatus::AlreadyInBase`] when `head`'s whole change
6019    /// is already on `tip` under other commit ids ([`crate::already`]); returns
6020    /// whether it did.
6021    ///
6022    /// Checked only when the base is ahead of the branch. A failing check is
6023    /// "not proven" - the ordinary rebase path then decides - never a reason to
6024    /// stop the run.
6025    ///
6026    /// The remote copy of the branch is held to the same standard as the local
6027    /// one: if it carries a tip this worktree does not, that tip must itself be
6028    /// proven in the base, or nothing is settled (a pull request would
6029    /// otherwise be closed over commits nobody checked). The pull request is
6030    /// closed *before* the terminal status is saved; if that fails for a
6031    /// reason other than a refusal (no network, a `gh` error) the run is left
6032    /// `Blocked` with the reason as its conflict, which a resume retries -
6033    /// the same recovery a phantom conflict gets.
6034    async fn settle_already_in(
6035        &mut self,
6036        branch: &str,
6037        tip: &str,
6038        head: &str,
6039        attempts: usize,
6040        behind: usize,
6041    ) -> Result<bool> {
6042        let repo = self.state.repo.clone();
6043        let remote = self.state.config.merge.remote.clone();
6044        let start = self.state.base_commit.clone();
6045        let evidence = match crate::already::classify(&repo, tip, head, Some(&start)).await {
6046            Ok(Some(e)) => e,
6047            Ok(None) => return Ok(false),
6048            Err(e) => {
6049                tracing::warn!("already-in-base check for {branch}: {e:#}");
6050                return Ok(false);
6051            }
6052        };
6053        let mut verified = vec![head.to_owned()];
6054        let public_branch = self.state.pr_branch().unwrap_or(branch).to_owned();
6055        let fetched = git::fetch(&repo, &remote, &public_branch).await;
6056        if matches!(&fetched, Ok(o) if o.ok())
6057            && let Ok(theirs) = git::rev_parse(&repo, &format!("{remote}/{public_branch}")).await
6058            && theirs != head
6059        {
6060            match crate::already::classify(&repo, tip, &theirs, Some(&start)).await {
6061                Ok(Some(_)) => verified.push(theirs),
6062                _ => return Ok(false),
6063            }
6064        }
6065        let base_branch = self.state.base_branch.clone();
6066        let message = format!(
6067            "{branch} is already in {remote}/{base_branch} as {} ({} match); nothing left to \
6068             land",
6069            evidence.names(),
6070            evidence.proof.as_str()
6071        );
6072        let closed =
6073            crate::land::close_superseded_pr(&mut self.state, &public_branch, &evidence, &verified)
6074                .await;
6075        match closed {
6076            Ok(Ok(url)) => self
6077                .state
6078                .event("land", format!("closed {url}: superseded on {base_branch}")),
6079            Ok(Err(why)) => self
6080                .state
6081                .event("land", format!("did not close a pull request: {why}")),
6082            Err(e) => {
6083                let why = format!(
6084                    "{branch} is already in {remote}/{base_branch}, but its pull request could \
6085                     not be closed ({e:#}); resume to retry"
6086                );
6087                self.state.status = RunStatus::Blocked;
6088                self.state.base_sync = Some(BaseSync {
6089                    tip: tip.to_owned(),
6090                    behind,
6091                    attempts,
6092                    conflict: Some(why.clone()),
6093                    already_in: None,
6094                });
6095                self.state.event("land", why);
6096                self.hold_base_sync_kind(crate::base_sync_recovery::StopKind::ClosingIntegratedPr)
6097                    .await?;
6098                self.state.save()?;
6099                return Ok(true);
6100            }
6101        }
6102        self.state.status = RunStatus::AlreadyInBase;
6103        self.state.base_sync_recovery.stop = None;
6104        self.state.base_sync = Some(BaseSync {
6105            tip: tip.to_owned(),
6106            behind,
6107            attempts,
6108            conflict: None,
6109            already_in: Some(evidence),
6110        });
6111        self.state.event("land", message);
6112        self.state.save()?;
6113        self.settle_questions();
6114        Ok(true)
6115    }
6116
6117    /// The commit review and gate diff against: the tip [`Self::sync_to_base`]
6118    /// last landed the winner on, once it has run, else the commit the run
6119    /// branched from.
6120    ///
6121    /// Only [`Self::review_loop`] reads this. `prep`, `judge`, `deliberate`
6122    /// and `vote` all happen before there is a winner to rebase, so they
6123    /// compare every candidate against the branch point on purpose, and a
6124    /// base that moves after they are already done cannot change an answer
6125    /// they already gave.
6126    fn landing_base(&self) -> String {
6127        self.state
6128            .base_sync
6129            .as_ref()
6130            .map_or_else(|| self.state.base_commit.clone(), |s| s.tip.clone())
6131    }
6132
6133    // ------------------------------------------------------- operator fix
6134
6135    /// Route specific, already-recorded review findings to a fixer for a
6136    /// targeted, out-of-band fix on the winning branch — `magi fix`'s own
6137    /// entry point.
6138    ///
6139    /// Distinct from `review_loop`'s own fix step in three ways: it never
6140    /// runs a reviewer wave, it never spends review-round budget, and what
6141    /// happened is recorded as an [`OperatorFixRequest`] appended to
6142    /// [`RunState::operator_fixes`], never folded into a [`ReviewRound`] —
6143    /// see `run::SCHEMA`'s doc for schema 9 on why a reviewer's own severity
6144    /// and vote must never be rewritten to look like a manufactured blocking
6145    /// verdict.
6146    ///
6147    /// Only meaningful once review has actually concluded: `Ready` (handed
6148    /// off with findings still open, or simply concluded clean while minor
6149    /// findings sat unaddressed) or `Blocked` (round budget spent, or the
6150    /// gate failed). Everything else is refused: a run still in progress
6151    /// should simply be resumed, and a `Merged` run's branch has already
6152    /// landed — reopening *this* run's own record cannot change that, so the
6153    /// answer there is a fresh `magi review <branch>`.
6154    ///
6155    /// A real commit here re-verifies through a fresh, ordinary review-only
6156    /// run on the same branch ([`Self::review`]) rather than reopening this
6157    /// run's own `review_loop`: once any round in this run's history went
6158    /// clean, `review_conclusion` treats that as permanent by design (the
6159    /// same purity `gate`/`merge` rely on for safe reentry), so there is no
6160    /// way to force one more genuine reviewer wave out of *this* run without
6161    /// either rewriting history or weakening that guarantee for every other
6162    /// caller. A review-only run costs nothing extra — no implementation, no
6163    /// judging, no vote — and exercises the exact same review → verify →
6164    /// gate → (human) merge path, unmodified.
6165    pub async fn fix_selected(
6166        &mut self,
6167        ids: &[String],
6168        reason: &str,
6169        allow_stale: bool,
6170    ) -> Result<()> {
6171        let reason = reason.trim();
6172        if reason.is_empty() {
6173            bail!("a fix request needs a reason — that is the operator's own record of why");
6174        }
6175        if ids.is_empty() {
6176            bail!("no finding id given");
6177        }
6178        if !matches!(self.state.status, RunStatus::Ready | RunStatus::Blocked) {
6179            bail!(
6180                "run {} is `{}`; only a `ready` or `blocked` run — one whose review \
6181                 has already concluded — can be given a targeted fix. A run still \
6182                 in progress should simply be resumed; a `merged` run's branch has \
6183                 already landed, so its answer is a fresh `magi review <branch>`, \
6184                 not reopening this run's own record",
6185                self.state.id,
6186                self.state.status.as_str()
6187            );
6188        }
6189        // Reassignment shares this claim through its atomic task save. Check
6190        // the canonical run identity inside it, before preflight or any await.
6191        let _claim = FixClaim::acquire(&self.state.dir())?;
6192        crate::task_reassign::ensure_current(&crate::queue::Queue::open(), &self.state.id)?;
6193        let Some(winner) = self.state.winner().cloned() else {
6194            bail!("run {} has no winning candidate to fix", self.state.id);
6195        };
6196        if !git::branch_exists(&self.state.repo, &winner.branch).await? {
6197            bail!(
6198                "branch `{}` no longer exists; this run cannot be extended",
6199                winner.branch
6200            );
6201        }
6202        let home = crate::run::home();
6203        if crate::daemon::is_working_on(&home, &self.state.id, Timestamp::now()) {
6204            bail!(
6205                "run {} is currently being worked on by another magi process",
6206                self.state.id
6207            );
6208        }
6209        // Resolve every id before spending anything — an unknown id refuses
6210        // the whole request rather than silently dropping it — and dedup
6211        // while keeping the operator's own order.
6212        let mut seen = BTreeSet::new();
6213        let mut findings = Vec::new();
6214        let mut missing = Vec::new();
6215        for id in ids {
6216            if !seen.insert(id.clone()) {
6217                continue;
6218            }
6219            match self.state.finding(id) {
6220                Some((round, rec, f)) => findings.push(OperatorFixFinding {
6221                    id: f.id.clone(),
6222                    severity: f.severity,
6223                    reviewer_vote: rec.vote,
6224                    round: round.round,
6225                    round_head: round.head.clone(),
6226                    reviewer: rec.reviewer,
6227                    agent: rec.agent.clone(),
6228                    file: f.file.clone(),
6229                    line: f.line,
6230                    title: f.title.clone(),
6231                    detail: f.detail.clone(),
6232                    outcome: OperatorFixOutcome::Pending,
6233                }),
6234                None => missing.push(id.clone()),
6235            }
6236        }
6237        if !missing.is_empty() {
6238            bail!(
6239                "unknown finding id(s): {}; nothing was changed",
6240                missing.join(", ")
6241            );
6242        }
6243
6244        let head_at_request = git::rev_parse(&self.state.repo, &winner.branch).await?;
6245        let stale_details: Vec<(String, String)> = findings
6246            .iter()
6247            .filter(|f| f.round_head != head_at_request)
6248            .map(|f| (f.id.clone(), f.round_head.clone()))
6249            .collect();
6250        let stale = !stale_details.is_empty();
6251        if stale && !allow_stale {
6252            bail!(
6253                "the branch has moved since some finding(s) were raised — {} — now \
6254                 at {}; pass --allow-stale to fix anyway, or re-run review first",
6255                stale_details
6256                    .iter()
6257                    .map(|(id, head)| format!("{id} (raised against {})", short(head)))
6258                    .collect::<Vec<_>>()
6259                    .join(", "),
6260                short(&head_at_request)
6261            );
6262        }
6263
6264        let request = OperatorFixRequest {
6265            requested_at: Timestamp::now(),
6266            reason: reason.to_owned(),
6267            findings,
6268            head_at_request: head_at_request.clone(),
6269            allow_stale,
6270            stale,
6271            fix: None,
6272            result_head: None,
6273            follow_up_review_run: None,
6274        };
6275        self.state.event(
6276            "fix",
6277            format!(
6278                "operator requested a targeted fix on {} finding(s) ({}): {reason}",
6279                request.findings.len(),
6280                request
6281                    .findings
6282                    .iter()
6283                    .map(|f| f.id.as_str())
6284                    .collect::<Vec<_>>()
6285                    .join(", "),
6286            ),
6287        );
6288        // Recorded now, before any worktree work or the fixer call itself —
6289        // and re-saved at each checkpoint below: a crash at any point after
6290        // this (mid fixer call, mid follow-up review) must not lose the fact
6291        // that this was requested, for which findings, and why. Everything
6292        // past this point reads and writes through `request_index` rather
6293        // than a local variable, since `request` itself is moved here.
6294        self.state.operator_fixes.push(request);
6295        self.state.save()?;
6296        let request_index = self.state.operator_fixes.len() - 1;
6297
6298        // A fresh, dedicated worktree for this one call, never the winner's
6299        // own worktree in place: that one may already be gone (folded away),
6300        // and reusing it in place would leave the branch checked out there
6301        // when the follow-up review below tries to check it out again. Freed
6302        // immediately after, either way — but only once confirmed clean:
6303        // `worktree_remove` is a `git worktree remove --force`, which would
6304        // otherwise discard uncommitted work left there by the operator or
6305        // another process before this had a chance to even look at it.
6306        let repo = self.state.repo.clone();
6307        let coordination = self.coordinate(&repo).await?;
6308        if winner.worktree.exists() {
6309            // Lockfiles a rescue commit withheld stay untracked on purpose and
6310            // are already recorded; they are not the operator's work to protect.
6311            let dirty = git::git(
6312                &winner.worktree,
6313                &["status", "--porcelain", "--untracked-files=all"],
6314            )
6315            .await?;
6316            let only_withheld = dirty.lines().all(|l| {
6317                l.strip_prefix("?? ")
6318                    .is_some_and(|p| self.state.withheld.iter().any(|w| w.path == p))
6319            });
6320            if !only_withheld {
6321                bail!(
6322                    "`{}` has uncommitted changes; refusing to touch it — commit or \
6323                     discard them first",
6324                    winner.worktree.display()
6325                );
6326            }
6327            git::worktree_remove(&self.state.repo, &winner.worktree)
6328                .await
6329                .ok();
6330        }
6331        let fix_worktree = self.state.worktree_root().join("operator-fix");
6332        let fix_worktree_s = fix_worktree.to_string_lossy().to_string();
6333        git::git(
6334            &self.state.repo,
6335            &["worktree", "add", &fix_worktree_s, winner.branch.as_str()],
6336        )
6337        .await
6338        .with_context(|| format!("checking out `{}` for the fix", winner.branch))?;
6339        if !git::is_clean(&fix_worktree).await? {
6340            git::worktree_remove(&self.state.repo, &fix_worktree)
6341                .await
6342                .ok();
6343            bail!(
6344                "`{}` has uncommitted changes; refusing to start a fix on a dirty tree",
6345                winner.branch
6346            );
6347        }
6348        let repo_for_setup = self.state.repo.clone();
6349        let setup = match ensure_setup_config(&mut self.state, &repo_for_setup).await {
6350            Ok(()) => {
6351                worktree_setup::prepare(&self.state.config, &repo_for_setup, &fix_worktree).await
6352            }
6353            Err(e) => Err(e),
6354        };
6355        if let Err(e) = setup {
6356            git::worktree_remove(&self.state.repo, &fix_worktree)
6357                .await
6358                .ok();
6359            return Err(e);
6360        }
6361
6362        // This explicit operator request retries availability independently
6363        // of execute(); the previous call has already saved and cleaned up.
6364        self.state.resource_block = None;
6365
6366        let run_id = self.state.id.clone();
6367        let prompts = self.state.config.prompts.clone();
6368        let language = self.state.config.graph.language.clone();
6369        let sessions = self.state.config.graph.sessions;
6370        let artifacts = agent::artifacts_dir(&self.state.dir());
6371        let finding_list: Vec<Finding> = self.state.operator_fixes[request_index]
6372            .findings
6373            .iter()
6374            .map(|f| Finding {
6375                id: f.id.clone(),
6376                severity: f.severity,
6377                file: f.file.clone(),
6378                line: f.line,
6379                title: f.title.clone(),
6380                detail: f.detail.clone(),
6381            })
6382            .collect();
6383        let fix_prompt = prompt::operator_fix_in(
6384            &self.state.instruction,
6385            &finding_list,
6386            reason,
6387            &stale_details,
6388            &head_at_request,
6389            &language,
6390            &self.state.config.graph.github_language,
6391        );
6392        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6393        drop(coordination);
6394        let (job, seat, out) = self
6395            .ask_fixer(&winner, "fix", None, |spec, seat| SeatJob {
6396                prompt: fix_prompt.clone(),
6397                spec,
6398                seat,
6399                cwd: fix_worktree.clone(),
6400                timeout,
6401                allow_write: true,
6402                sessions,
6403                artifacts: artifacts.clone(),
6404                stem: "operator-fix".to_owned(),
6405                handover: None,
6406            })
6407            .await;
6408        let agent_id = seat.agent.clone();
6409
6410        let mut fix = FixRecord {
6411            agent: agent_id,
6412            addressed: Vec::new(),
6413            rejected: Vec::new(),
6414            notes: String::new(),
6415            committed: false,
6416            failed: None,
6417            duration_ms: 0,
6418            continuation: None,
6419        };
6420        let mut final_seat = seat.clone();
6421        match out {
6422            AgentOutcome::Ok(o) => {
6423                fix.duration_ms = o.duration_ms;
6424                let parsed = verdict::extract_json::<FixReport>(&o.text);
6425                let incomplete_reason = match &parsed {
6426                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
6427                        "the reply parsed, but it reported a command whose own CLI \
6428                         never confirmed an exit status"
6429                            .to_owned(),
6430                    ),
6431                    Ok(_) => None,
6432                    Err(e) => Some(e.to_string()),
6433                };
6434                match incomplete_reason {
6435                    None => {
6436                        let report = parsed.expect("checked Ok above");
6437                        fix.addressed = report.addressed;
6438                        fix.rejected = report.rejected;
6439                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
6440                    }
6441                    Some(reason) => {
6442                        let (resumed_seat, resolved, failure, cont) = self
6443                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, 0)
6444                            .await;
6445                        fix.duration_ms += cont.cumulative_wait_ms;
6446                        fix.continuation = Some(cont);
6447                        final_seat = resumed_seat;
6448                        match resolved {
6449                            Some(report) => {
6450                                fix.addressed = report.addressed;
6451                                fix.rejected = report.rejected;
6452                                fix.notes =
6453                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
6454                            }
6455                            None => fix.failed = failure,
6456                        }
6457                    }
6458                }
6459            }
6460            AgentOutcome::Dropped(o) => {
6461                fix.duration_ms = o.duration_ms;
6462                let why = o
6463                    .dropped
6464                    .as_ref()
6465                    .map(|d| d.why.as_str())
6466                    .unwrap_or("the CLI ended the stream without delivering its answer");
6467                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
6468            }
6469            AgentOutcome::Quota(o) => {
6470                self.state.quota.push(QuotaLoss::from_report(
6471                    final_seat.key.clone(),
6472                    "fix",
6473                    o.quota.as_ref(),
6474                ));
6475                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
6476            }
6477            AgentOutcome::Failed(e) => fix.failed = Some(e),
6478            AgentOutcome::ResourceBlocked(e) => {
6479                fix.failed = Some(format!("the fixer was not started: {e}"));
6480            }
6481        }
6482        // Even a deferred call must rescue edits, release the worktree and
6483        // finalize its record below so an explicit `magi fix` can retry.
6484        if fix.continuation.is_none() {
6485            fix.continuation = Some(ContinuationRecord::not_needed());
6486        }
6487        self.state.seats.insert(final_seat.key.clone(), final_seat);
6488
6489        let rescue_message = format!(
6490            "magi: operator-selected fix ({}) (uncommitted work)",
6491            self.state.operator_fixes[request_index]
6492                .findings
6493                .iter()
6494                .map(|f| f.id.as_str())
6495                .collect::<Vec<_>>()
6496                .join(", ")
6497        );
6498        let repo = self.state.repo.clone();
6499        let coordination = self.coordinate(&repo).await?;
6500        if let Ok(r) = git::rescue_commit(&fix_worktree, &rescue_message).await {
6501            self.state.note_withheld("fix", &r.withheld);
6502        }
6503        let after = git::rev_parse(&fix_worktree, "HEAD").await?;
6504        fix.committed = after != head_at_request;
6505        git::worktree_remove(&self.state.repo, &fix_worktree)
6506            .await
6507            .ok();
6508        drop(coordination);
6509
6510        self.state.event(
6511            "fix",
6512            match &fix.failed {
6513                Some(reason) => format!(
6514                    "operator fix: adoption report was lost ({reason}); {}",
6515                    if fix.committed {
6516                        "committed"
6517                    } else {
6518                        "NO new commit"
6519                    }
6520                ),
6521                None => format!(
6522                    "operator fix: {} addressed, {} rejected, {}",
6523                    fix.addressed.len(),
6524                    fix.rejected.len(),
6525                    if fix.committed {
6526                        "committed"
6527                    } else {
6528                        "NO new commit"
6529                    }
6530                ),
6531            },
6532        );
6533
6534        // Every selected finding gets an outcome — never left `Pending` once
6535        // the fixer's own turn is over. A report that never came back at all
6536        // marks every one of them `Unreported`, not silently "not addressed":
6537        // quota, a dropped stream, or an exhausted continuation are gaps in
6538        // the report, not evidence about the finding itself (see [`SCHEMA`]'s
6539        // doc for schema 9 and [`OperatorFixOutcome::Unreported`]).
6540        for f in &mut self.state.operator_fixes[request_index].findings {
6541            f.outcome = if fix.failed.is_some() {
6542                OperatorFixOutcome::Unreported
6543            } else if fix.addressed.contains(&f.id) {
6544                OperatorFixOutcome::Addressed
6545            } else if let Some(r) = fix.rejected.iter().find(|r| r.id == f.id) {
6546                OperatorFixOutcome::Rejected { why: r.why.clone() }
6547            } else {
6548                OperatorFixOutcome::Unreported
6549            };
6550        }
6551
6552        let committed = fix.committed;
6553        if committed {
6554            self.state.operator_fixes[request_index].result_head = Some(after.clone());
6555        }
6556        self.state.operator_fixes[request_index].fix = Some(fix);
6557        // Saved again now that the fixer's own outcome is final, on top of
6558        // the save right after the request was first pushed above.
6559        self.state.save()?;
6560        if self.state.resource_block.is_some() {
6561            return self.stop_for_resource();
6562        }
6563
6564        if committed {
6565            self.state.event(
6566                "fix",
6567                format!(
6568                    "operator fix committed {}; opening a follow-up review-only run",
6569                    short(&after)
6570                ),
6571            );
6572            // The operator asked for the fix, and the follow-up serves whatever
6573            // task the run it follows served.
6574            let origin =
6575                Origin::operator().serving(self.state.origin.as_ref().and_then(|o| o.task.clone()));
6576            match Self::review(
6577                &self.state.repo,
6578                &winner.branch,
6579                self.state.config.clone(),
6580                origin,
6581            )
6582            .await
6583            {
6584                Ok(mut follow_up) => {
6585                    follow_up.state.event(
6586                        "start",
6587                        format!(
6588                            "requested by an operator fix on run {} for finding(s) {}",
6589                            self.state.id,
6590                            self.state.operator_fixes[request_index]
6591                                .findings
6592                                .iter()
6593                                .map(|f| f.id.as_str())
6594                                .collect::<Vec<_>>()
6595                                .join(", "),
6596                        ),
6597                    );
6598                    follow_up.state.save()?;
6599                    let follow_up_id = follow_up.state.id.clone();
6600                    // The follow-up is a run like any other: it belongs to the
6601                    // task of the run it follows, or to one filed for it.
6602                    let adopted = match crate::direct::adopt(
6603                        &follow_up.state,
6604                        self.state.origin.as_ref().and_then(|o| o.task.as_deref()),
6605                    ) {
6606                        Ok(a) => a,
6607                        Err(e) => {
6608                            // An ownerless run must not spend agent calls; it
6609                            // stays saved, and `magi run --resume` adopts it.
6610                            self.state.event(
6611                                "fix",
6612                                format!(
6613                                    "follow-up review {follow_up_id} got no owning task and was not executed: {e:#}"
6614                                ),
6615                            );
6616                            self.state.save()?;
6617                            return Ok(());
6618                        }
6619                    };
6620                    let executed = follow_up.execute().await;
6621                    let failure = executed.as_ref().err().map(|e| format!("{e:#}"));
6622                    if let Some(a) = adopted {
6623                        a.finish(&follow_up.state, executed);
6624                    }
6625                    if let Some(e) = failure {
6626                        self.state.event(
6627                            "fix",
6628                            format!(
6629                                "follow-up review {follow_up_id} did not complete cleanly: {e:#}"
6630                            ),
6631                        );
6632                    }
6633                    self.state.operator_fixes[request_index].follow_up_review_run =
6634                        Some(follow_up_id);
6635                }
6636                Err(e) => {
6637                    self.state.event(
6638                        "fix",
6639                        format!("committed the fix but could not open a follow-up review: {e:#}"),
6640                    );
6641                }
6642            }
6643            self.state.save()?;
6644        }
6645
6646        Ok(())
6647    }
6648
6649    // --------------------------------------------------------------- review
6650
6651    /// Ask the fixer chain once for one fix call: the agents [`fixer::attempts`]
6652    /// names, in order, each at most once, moving on only when the call
6653    /// advances (an error, a quota hit or nothing usable - the same decision
6654    /// point `agent::chain_advances` is for the other chained roles).
6655    ///
6656    /// Each agent gets its own seat from `Runner::seat`: the same agent
6657    /// continues its conversation (the winner's own implementer seat when it
6658    /// is the winner's author, now that the competition is over), another
6659    /// takes a fresh one so the full prompt is sent again. A handover is
6660    /// recorded under `node`, which is what makes the fallback stick for the
6661    /// rest of the run (see `crate::fixer`). Only the last attempt's outcome
6662    /// is returned, so an exhausted chain reads exactly like a single failed
6663    /// fixer: one quota loss, the same wording, the same refund. An earlier
6664    /// attempt's edits are left in the tree and judged, with the final
6665    /// attempt's, by what git says afterwards.
6666    ///
6667    /// Returns the job that produced the outcome, for `continue_fix_report`.
6668    async fn ask_fixer(
6669        &mut self,
6670        winner: &Candidate,
6671        node: &'static str,
6672        round: Option<usize>,
6673        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6674    ) -> (SeatJob, SeatState, AgentOutcome) {
6675        let attempts = fixer::attempts(&self.state, &self.roles, winner);
6676        self.ask_fixer_among(attempts, node, round, None, build)
6677            .await
6678    }
6679
6680    /// [`Self::ask_fixer`] over an explicit attempt list. `restore` is a
6681    /// worktree and a commit: before the next agent is asked after one that
6682    /// did not satisfy the call, the worktree is put back on that commit, so
6683    /// a failed agent's partial edits are not carried into the next one.
6684    async fn ask_fixer_among(
6685        &mut self,
6686        attempts: Vec<(AgentSpec, String)>,
6687        node: &'static str,
6688        round: Option<usize>,
6689        restore: Option<(&Path, &str)>,
6690        build: impl Fn(AgentSpec, SeatState) -> SeatJob,
6691    ) -> (SeatJob, SeatState, AgentOutcome) {
6692        let run_id = self.state.id.clone();
6693        let prompts = self.state.config.prompts.clone();
6694        let cache = self.state.config.cache_dir();
6695        let ids: Vec<String> = attempts.iter().map(|(s, _)| s.id.clone()).collect();
6696        let mut last = None;
6697        for (i, (spec, key)) in attempts.into_iter().enumerate() {
6698            let seat = self.seat(&key, &spec.id);
6699            let mut job = build(spec.clone(), seat);
6700            if i > 0 {
6701                job.stem = format!("{}-{}", job.stem, spec.id);
6702            }
6703            let ctx = WaveCtx {
6704                carry_seats: false,
6705                pauses: Some((&self.pause, &self.interrupt)),
6706                run: &run_id,
6707                node,
6708                prompts: &prompts,
6709                cache: cache.as_deref(),
6710                round,
6711            };
6712            let (seat, out) =
6713                run_one(job.clone(), Arc::clone(&self.sem), &ctx, &mut self.state, 0).await;
6714            // A resource block never advances the chain: the chosen agent
6715            // was not asked, so it must be the one asked again on resume.
6716            let advances = match &out {
6717                AgentOutcome::Ok(o) => agent::output_advances(o),
6718                AgentOutcome::ResourceBlocked(_) => false,
6719                _ => true,
6720            };
6721            if let Some(next) = ids.get(i + 1)
6722                && advances
6723            {
6724                self.state.seats.insert(seat.key.clone(), seat.clone());
6725                let class =
6726                    FailClass::of(&out).unwrap_or_else(|| FailClass::Other("unusable".to_owned()));
6727                let reason = match &out {
6728                    AgentOutcome::Ok(_) => "nothing usable".to_owned(),
6729                    other => fail_reason(other),
6730                };
6731                // A failed agent's edits must not reach the next one, and a
6732                // restore that did not work ends the chain: the call fails
6733                // and the caller's own (confirmed) restore decides.
6734                if let Some((wt, rev)) = restore
6735                    && let Err(e) = self.graft_restore(wt, rev).await
6736                {
6737                    let why =
6738                        format!("could not restore the worktree before the next fixer: {e:#}");
6739                    tracing::warn!("{why}");
6740                    let out = if e.is::<crate::repo_lock::ResourceBlocked>() {
6741                        AgentOutcome::ResourceBlocked(why)
6742                    } else {
6743                        AgentOutcome::Failed(why)
6744                    };
6745                    last = Some((job, seat, out));
6746                    break;
6747                }
6748                record_handover(
6749                    &mut self.state,
6750                    node,
6751                    &seat.key,
6752                    &spec.id,
6753                    next,
6754                    &class,
6755                    &reason,
6756                );
6757                continue;
6758            }
6759            last = Some((job, seat, out));
6760            break;
6761        }
6762        let (job, seat, out) = last.expect("the fixer chain always has an entry");
6763        if let AgentOutcome::Quota(output) = &out
6764            && node != "graft"
6765        {
6766            // Graft is best effort: its caller records the loss and restores
6767            // the pre-graft tree, then proceeds to review and gate.
6768            // Every fallback was tried. Machine outages cannot be repaired by
6769            // spending a fix round; use each caller's resumable resource path.
6770            let loss = QuotaLoss::from_report(seat.key.clone(), node, output.quota.as_ref());
6771            block_machine_outage(&mut self.state, node, std::slice::from_ref(&loss));
6772            self.state.quota.push(loss);
6773            self.state.seats.insert(seat.key.clone(), seat.clone());
6774            self.state.status = RunStatus::Blocked;
6775            return (
6776                job,
6777                seat,
6778                AgentOutcome::ResourceBlocked(
6779                    "fixer unavailable; restore agent availability and resume".into(),
6780                ),
6781            );
6782        }
6783        (job, seat, out)
6784    }
6785
6786    /// The one repair a verification-only failure earns (see
6787    /// [`crate::recovery`]), before the first review round of a run that took
6788    /// a branch over.
6789    ///
6790    /// Saved as started *before* the fixer is called, so a resume never
6791    /// repairs twice; a start found on re-entry is rolled back to the commit
6792    /// it began from. A failed or unusable fixer leaves the tree as it was and
6793    /// the review goes on over it, so the first round's own verification
6794    /// reports the failure again. The failure is never carried into a round as
6795    /// a result: verification is re-run on whatever commit this leaves.
6796    async fn recovery_repair(&mut self, winner: &Candidate) -> Result<()> {
6797        use crate::recovery::{Mode, Repair};
6798        if !self.state.reviews.is_empty() {
6799            return Ok(());
6800        }
6801        let Some(mut rec) = self.state.recovery.clone() else {
6802            return Ok(());
6803        };
6804        if rec.mode != Mode::VerifyRepair {
6805            return Ok(());
6806        }
6807        let wt = winner.worktree.clone();
6808        let mut head = git::rev_parse(&wt, "HEAD").await?;
6809        match rec.repair.clone() {
6810            Repair::Done { .. } => return Ok(()),
6811            Repair::WaitingForRestore { before } => {
6812                self.graft_restore(&wt, &before).await?;
6813                head = before;
6814                rec.repair = Repair::NotStarted;
6815                self.state.recovery = Some(rec.clone());
6816                self.state.save()?;
6817            }
6818            Repair::Started { before } => {
6819                // Interrupted: the unfinished edits are not trusted.
6820                // An unconfirmed rollback blocks the run with `Started` kept,
6821                // so a resume retries it; the tree is never reviewed as is.
6822                if let Err(e) = self.graft_restore(&wt, &before).await {
6823                    if e.is::<crate::repo_lock::ResourceBlocked>() {
6824                        return Err(e);
6825                    }
6826                    return self
6827                        .stop_recovery(rec, &before, format!("interrupted; rollback failed: {e:#}"))
6828                        .await;
6829                }
6830                rec.repair = Repair::Done {
6831                    before: before.clone(),
6832                    after: before,
6833                    committed: false,
6834                    failed: Some("interrupted; rolled back".to_owned()),
6835                };
6836                self.state
6837                    .event("recovery", "the repair was interrupted and rolled back");
6838                self.state.recovery = Some(rec);
6839                self.state.save()?;
6840                return Ok(());
6841            }
6842            Repair::NotStarted => {}
6843        }
6844        if crate::recovery::recheck(&mut rec, &head) {
6845            self.state.event("recovery", rec.summary());
6846            self.state.recovery = Some(rec);
6847            self.state.save()?;
6848            return Ok(());
6849        }
6850        rec.repair = Repair::Started {
6851            before: head.clone(),
6852        };
6853        self.state.recovery = Some(rec.clone());
6854        self.state.save()?;
6855
6856        let language = self.state.config.graph.language.clone();
6857        let sessions = self.state.config.graph.sessions;
6858        let artifacts = agent::artifacts_dir(&self.state.dir());
6859        let fix_prompt = prompt::recovery_fix(
6860            &self.state.instruction,
6861            &rec,
6862            self.state.config.graph.review_rounds,
6863            &language,
6864        );
6865        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
6866        self.state.event(
6867            "recovery",
6868            format!(
6869                "repair: the fixer starts from the failed verification on {}",
6870                short(&head)
6871            ),
6872        );
6873        let (_job, seat, out) = self
6874            .ask_fixer(winner, "fix", None, |spec, seat| SeatJob {
6875                prompt: fix_prompt.clone(),
6876                spec,
6877                seat,
6878                cwd: wt.clone(),
6879                timeout,
6880                allow_write: true,
6881                sessions,
6882                artifacts: artifacts.clone(),
6883                stem: "fix-recovery".to_owned(),
6884                handover: None,
6885            })
6886            .await;
6887        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
6888            rec.repair = Repair::WaitingForRestore { before: head };
6889            self.state.recovery = Some(rec);
6890            return self.stop_for_resource();
6891        }
6892        let failed = match &out {
6893            AgentOutcome::Ok(_) | AgentOutcome::ResourceBlocked(_) => None,
6894            AgentOutcome::Quota(o) => {
6895                self.state.quota.push(QuotaLoss::from_report(
6896                    seat.key.clone(),
6897                    "fix",
6898                    o.quota.as_ref(),
6899                ));
6900                Some("rate limited (quota); the fixer could not run".to_owned())
6901            }
6902            AgentOutcome::Dropped(_) => Some("the CLI dropped the stream".to_owned()),
6903            AgentOutcome::Failed(e) => Some(e.clone()),
6904        };
6905        self.state.seats.insert(seat.key.clone(), seat);
6906        let coordination = self.coordinate(&wt).await?;
6907        let failed = match failed {
6908            // The agent's partial edits are not kept.
6909            Some(why) => {
6910                if let Err(e) = git::restore_to(&wt, &head).await {
6911                    return self
6912                        .stop_recovery(rec, &head, format!("{why}; restore failed: {e:#}"))
6913                        .await;
6914                }
6915                Some(why)
6916            }
6917            None => {
6918                if let Ok(r) = git::rescue_commit(
6919                    &wt,
6920                    "magi: recovery repair of the failed verification (uncommitted work)",
6921                )
6922                .await
6923                {
6924                    self.state.note_withheld("fix", &r.withheld);
6925                }
6926                None
6927            }
6928        };
6929        let after = git::rev_parse(&wt, "HEAD").await?;
6930        drop(coordination);
6931        self.state.event(
6932            "recovery",
6933            match &failed {
6934                Some(why) => format!("repair: nothing kept ({why})"),
6935                None if after != head => {
6936                    format!("repair: committed {}", short(&after))
6937                }
6938                None => "repair: the fixer made no commit".to_owned(),
6939            },
6940        );
6941        rec.repair = Repair::Done {
6942            before: head.clone(),
6943            committed: after != head,
6944            after,
6945            failed,
6946        };
6947        self.state.recovery = Some(rec);
6948        self.state.save()?;
6949        Ok(())
6950    }
6951
6952    /// A repair whose rollback could not be confirmed: the run is blocked and
6953    /// resumable, and the unconfirmed state is never reviewed.
6954    async fn stop_recovery(
6955        &mut self,
6956        mut rec: crate::recovery::RecoveryRecord,
6957        head: &str,
6958        why: String,
6959    ) -> Result<()> {
6960        // `Started` stays recorded: a resume retries the restore.
6961        rec.repair = crate::recovery::Repair::Started {
6962            before: head.to_owned(),
6963        };
6964        self.state.recovery = Some(rec);
6965        self.state
6966            .event("recovery", format!("repair could not be undone: {why}"));
6967        self.state.status = RunStatus::Blocked;
6968        self.state.save()?;
6969        bail!("recovery repair could not be rolled back: {why}")
6970    }
6971
6972    async fn review_loop(&mut self) -> Result<()> {
6973        if self.hold_base_sync().await? {
6974            return Ok(());
6975        }
6976        loop {
6977            if self
6978                .state
6979                .base_sync
6980                .as_ref()
6981                .is_some_and(|s| s.conflict.is_some())
6982            {
6983                return Ok(());
6984            }
6985            if self.catchup_repairable() {
6986                let record = self.state.reviews.last().expect("repairable round").clone();
6987                let winner = self
6988                    .state
6989                    .winner()
6990                    .cloned()
6991                    .context("catch-up repair has no winner")?;
6992                if !self.fix_review_round(&winner, record).await? {
6993                    return Ok(());
6994                }
6995            }
6996            self.review_loop_pass().await?;
6997            if !self.catchup_repairable() {
6998                return Ok(());
6999            }
7000        }
7001    }
7002
7003    /// Derive an owed catch-up repair from durable round evidence, including
7004    /// reentry after a crash before the fixer was dispatched.
7005    fn catchup_repairable(&self) -> bool {
7006        self.state.reviews.len() < self.state.config.graph.review_rounds
7007            && self.state.reviews.last().is_some_and(|r| {
7008                !r.clean
7009                    && r.blocking == 0
7010                    && r.fix.is_none()
7011                    && r.e2e_status() == E2eStatus::Failed
7012            })
7013    }
7014
7015    async fn review_loop_pass(&mut self) -> Result<()> {
7016        // A base that would not rebase is a person's decision, not a review
7017        // round: nothing here would change the answer, and reviewers and a
7018        // fixer would be spending real budget on a tree that cannot land
7019        // regardless of what they find.
7020        if self
7021            .state
7022            .base_sync
7023            .as_ref()
7024            .is_some_and(|s| s.conflict.is_some())
7025        {
7026            return Ok(());
7027        }
7028        // Attribution for every agent this node spawns: `MAGI_RUN` lets a task the
7029        // agent files with `magi task add` name the run that paid for it. The
7030        // prompt overlay is cloned alongside it because the waves borrow it
7031        // while `self` is mutably borrowed by the node's own bookkeeping.
7032        let run_id = self.state.id.clone();
7033        let prompts = self.state.config.prompts.clone();
7034        let Some(winner) = self.state.winner().cloned() else {
7035            return Ok(());
7036        };
7037        let granted_review = self
7038            .state
7039            .repair_grants
7040            .last()
7041            .filter(|g| g.phase == crate::repair_budget::Phase::Reviewing)
7042            .cloned();
7043        let fresh_review = granted_review
7044            .as_ref()
7045            .is_some_and(|g| self.state.reviews.len() == g.review_start);
7046        let max_rounds = granted_review
7047            .as_ref()
7048            .map_or(self.state.config.graph.review_rounds, |g| {
7049                g.review_start.saturating_add(1)
7050            });
7051        // A clean round, an exhausted round budget, or a stalled tree (see
7052        // `STAGNANT_LIMIT`) are all already-decided conclusions the moment
7053        // they are recorded — recomputed here, not read off `status`, so a
7054        // reentry into a run that already stopped restates the identical
7055        // verdict instead of silently handing back whatever an earlier node
7056        // in this same walk clobbered `status` to (a solo-candidate
7057        // `judge`/`deliberate` skip rewrites it on every reentry). The loop
7058        // below runs an empty range once the budget is spent, and would
7059        // otherwise fall through without touching `status` at all.
7060        if let Some(status) =
7061            review_conclusion(&self.state.reviews, max_rounds).filter(|_| !fresh_review)
7062        {
7063            // A reentry after a crash between the last round's save and
7064            // `stop_reviewing` reaches the hand-off here, not there.
7065            if status == RunStatus::Gating {
7066                self.record_contested_handoff();
7067            }
7068            self.state.status = status;
7069            self.state.save()?;
7070            return Ok(());
7071        }
7072        self.state.status = RunStatus::Reviewing;
7073        // A last recorded round whose own verification never resolved
7074        // (`ResourceBlocked` — the shared build cache, not the patch) is
7075        // never a concluded round, whatever the round budget says: starting
7076        // a fresh round on top of it would spend a whole new reviewer wave
7077        // re-reading an unchanged patch instead of just retrying the one
7078        // check that actually needs it, and once the budget is spent the
7079        // loop below has nothing left to do at all (its range is empty).
7080        // Retry that check directly instead, exactly the same retry
7081        // `stop_reviewing` already does for its own catch-up case.
7082        if !fresh_review
7083            && self.state.reviews.last().is_some_and(|r| {
7084                r.e2e_status() == E2eStatus::ResourceBlocked
7085                    || (r.e2e_status() == E2eStatus::Deferred && r.blocking == 0 && r.fix.is_none())
7086            })
7087        {
7088            let shell = self.state.config.shell();
7089            return self
7090                .stop_unresolved_review(
7091                    "the last round's own verification never resolved",
7092                    &shell,
7093                    &winner.worktree,
7094                )
7095                .await;
7096        }
7097
7098        if granted_review.is_none() {
7099            self.recovery_repair(&winner).await?;
7100        }
7101        if self.resource_halt()? {
7102            return Ok(());
7103        }
7104
7105        let repo = self.state.repo.clone();
7106        let root = self.state.worktree_root();
7107        let language = self.state.config.graph.language.clone();
7108        let sessions = self.state.config.graph.sessions;
7109        let artifacts = self.merge_repair_artifacts();
7110        let base = self.landing_base();
7111        let base_short = short(&base);
7112        let reviewers = self.roles.reviewers.clone();
7113        let shell = self.state.config.shell();
7114
7115        for round in (self.state.reviews.len() + 1)..=max_rounds {
7116            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7117            let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7118            let stat = git::diff_stat(&winner.worktree, &base, "HEAD").await?;
7119            // The prior round's own record, already persisted — never a
7120            // hand-carried variable of just its failing output: that is
7121            // exactly what let a round's e2e result drift out of sync with
7122            // which commit it was actually about (see `SCHEMA`'s doc for
7123            // schema 8). Judged against `head`, the commit reviewers are
7124            // about to look at now, so the summary always reads as "an
7125            // earlier head" here — this round's own patch has not been
7126            // checked yet.
7127            let prev_verification = self
7128                .state
7129                .reviews
7130                .last()
7131                .and_then(|r| r.verification_summary(&head));
7132            // Only the first round of a run that took a branch over: later
7133            // rounds have this run's own history.
7134            let prior_section = match (&self.state.recovery, round) {
7135                (Some(rec), 1) if self.state.reviews.is_empty() => {
7136                    let since = match (rec.mode, rec.reviewed_head.as_deref()) {
7137                        (crate::recovery::Mode::VerifyRepair, Some(reviewed)) => {
7138                            Some(crate::recovery::head_of(
7139                                &git::diff(&winner.worktree, reviewed, "HEAD")
7140                                    .await
7141                                    .unwrap_or_default(),
7142                                crate::recovery::SINCE_DIFF_MAX,
7143                            ))
7144                        }
7145                        _ => None,
7146                    };
7147                    Some(prompt::recovery_review(rec, since.as_deref()))
7148                }
7149                _ => None,
7150            };
7151
7152            // Each reviewer gets its own detached checkout of exactly this
7153            // commit: nobody can perturb the winner's tree, and the fixer can
7154            // keep working without racing a reviewer.
7155            let mut jobs = Vec::new();
7156            let coordination = if self.state.config.worktree.pre_clean.is_empty() {
7157                self.coordinate(&repo).await?
7158            } else {
7159                None
7160            };
7161            for (r, spec) in reviewers.iter().cloned().enumerate() {
7162                let wt = root.join(format!("review-{}", r + 1));
7163                // `clean -fdx` in a reset wipes setup's products, so every
7164                // round sets the seat up again.
7165                if wt.exists() {
7166                    crate::worktree_clean::reset(&mut self.state, &wt, Some(&head)).await?;
7167                } else {
7168                    let _lease = if self.state.config.worktree.pre_clean.is_empty() {
7169                        None
7170                    } else {
7171                        self.coordinate(&repo).await?
7172                    };
7173                    git::worktree_add_detached(&repo, &wt, &head).await?;
7174                }
7175                let _setup_lease = if self.state.config.worktree.pre_clean.is_empty() {
7176                    None
7177                } else {
7178                    self.coordinate(&repo).await?
7179                };
7180                self.setup_seat_worktree(&repo, &wt, false).await?;
7181                let seat_key = format!("review-{}", r + 1);
7182                // The seat starts the round on whoever answered it last, not
7183                // on the agent the spec names, so a failure is not re-paid.
7184                let spec = pick_start_spec(
7185                    &self.roles.reviewer_roster,
7186                    spec,
7187                    self.state.seat_history.get(&seat_key),
7188                );
7189                let seat = self.seat(&seat_key, &spec.id);
7190                jobs.push(SeatJob {
7191                    prompt: prompt::review(&prompt::ReviewCtx {
7192                        instruction: &self.state.instruction,
7193                        branch: &winner.branch,
7194                        base_short: &base_short,
7195                        stat: &stat,
7196                        patch: &patch,
7197                        verification: prev_verification.as_ref(),
7198                        prior: prior_section.as_deref(),
7199                        reviewers: reviewers.len(),
7200                        round,
7201                        rounds: max_rounds,
7202                        // A review-only run has no rankings, so nothing
7203                        // competed for this patch and the reviewer is told so.
7204                        competed: self.state.tally.as_ref().is_some_and(|t| t.rankings > 0),
7205                        lens: Lens::for_seat(r),
7206                        language: &language,
7207                        github_language: &self.state.config.graph.github_language,
7208                    }),
7209                    spec,
7210                    seat,
7211                    cwd: wt,
7212                    timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7213                    allow_write: false,
7214                    sessions,
7215                    artifacts: artifacts.clone(),
7216                    stem: format!("review-{round}-{}", r + 1),
7217                    handover: None,
7218                });
7219            }
7220
7221            drop(coordination);
7222            self.state.event(
7223                "review",
7224                format!(
7225                    "round {round}: {} reviewers on {}",
7226                    jobs.len(),
7227                    short(&head)
7228                ),
7229            );
7230            let mut quota_losses = Vec::new();
7231            let review_retries = self.state.config.graph.retries;
7232            let review_cache = self.state.config.cache_dir();
7233            let ctx = WaveCtx {
7234                carry_seats: true,
7235                pauses: Some((&self.pause, &self.interrupt)),
7236                run: &run_id,
7237                node: "review",
7238                prompts: &prompts,
7239                cache: review_cache.as_deref(),
7240                round: Some(round),
7241            };
7242            let results = ask_json_wave::<Review>(
7243                jobs,
7244                Arc::clone(&self.sem),
7245                review_retries,
7246                &self.roles.reviewer_roster,
7247                &ctx,
7248                &mut quota_losses,
7249                &mut self.state,
7250                &|_: &Review| Ok(()),
7251            )
7252            .await;
7253            if self.state.resource_block.is_some() {
7254                return self.stop_for_resource();
7255            }
7256            // Counted before the move below: how many of *this* round's
7257            // reviewer seats were lost to their own rate limit, as opposed to
7258            // a crash, a timeout, or unparsable output — see `round_is_clean`.
7259            let round_quota_missing = quota_losses.len();
7260            self.state.quota.extend(quota_losses);
7261
7262            let mut records = Vec::new();
7263            let mut all_findings = Vec::new();
7264            for (r, (seat, res, attempts)) in results.into_iter().enumerate() {
7265                let agent_id = seat.agent.clone();
7266                self.state.seats.insert(seat.key.clone(), seat);
7267                let mut record = ReviewRecord {
7268                    reviewer: r + 1,
7269                    agent: agent_id,
7270                    summary: String::new(),
7271                    findings: Vec::new(),
7272                    vote: None,
7273                    failed: None,
7274                    duration_ms: 0,
7275                    // Set for both outcomes: `failed: Some(_)` with
7276                    // `attempts > 0` is a seat every retry still lost, not a
7277                    // recovered one — only `failed: None` with `attempts > 0`
7278                    // reads as "answered after a nudge" (see this field's own
7279                    // doc).
7280                    attempts,
7281                };
7282                match res {
7283                    Ok((review, out)) => {
7284                        // Sanitized here, at the point every other piece of
7285                        // agent prose in this file is (candidate summaries,
7286                        // deliberation turns, vote reasons): a reviewer's own
7287                        // words are the one thing about it that could name
7288                        // it, and reconsideration below broadcasts this same
7289                        // summary and these same findings to every other
7290                        // seat on the panel.
7291                        record.summary =
7292                            blind::sanitize_prose(&review.summary, &self.state.config.blind);
7293                        record.vote = Some(review.vote);
7294                        record.duration_ms = out.duration_ms;
7295                        for (n, mut f) in review.findings.into_iter().enumerate() {
7296                            // ids are magi's, never the agent's: the fixer's
7297                            // adoption report is keyed by them.
7298                            f.id = format!("R{round}-{}-{}", r + 1, n + 1);
7299                            f.title = blind::sanitize_prose(&f.title, &self.state.config.blind);
7300                            f.detail = blind::sanitize_prose(&f.detail, &self.state.config.blind);
7301                            // `file` is agent-supplied prose too, never
7302                            // checked against the real tree — the same
7303                            // exposure `title`/`detail` above have, just in
7304                            // a field easy to forget because it looks like a
7305                            // path rather than free text.
7306                            f.file = f
7307                                .file
7308                                .map(|file| blind::sanitize_prose(&file, &self.state.config.blind));
7309                            all_findings.push(f.clone());
7310                            record.findings.push(f);
7311                        }
7312                        self.state.event(
7313                            "review",
7314                            format!(
7315                                "round {round}: reviewer {} voted {} with {} finding(s)",
7316                                r + 1,
7317                                review.vote.label(),
7318                                record.findings.len()
7319                            ),
7320                        );
7321                    }
7322                    Err(e) => {
7323                        record.failed = Some(e.to_string());
7324                        self.state.event(
7325                            "review",
7326                            format!("round {round}: reviewer {} produced nothing: {e}", r + 1),
7327                        );
7328                    }
7329                }
7330                records.push(record);
7331            }
7332
7333            // Tally the round's votes and, if they split, spend the one
7334            // round of reconsideration the split -> deliberate -> revote
7335            // shape `judge`/`vote` use for the panel, sized down to what a
7336            // read-only review round can afford: one round, and a revote
7337            // rather than an argument, because the panel already wrote its
7338            // reasoning down as findings the first time around.
7339            let initial_votes: Vec<ReviewVote> = records.iter().filter_map(|r| r.vote).collect();
7340            let vote_split =
7341                initial_votes.len() > 1 && !initial_votes.iter().all(|v| *v == initial_votes[0]);
7342            let mut reconsideration: Vec<ReviewRevoteRecord> = Vec::new();
7343            if vote_split {
7344                self.state.event(
7345                    "review",
7346                    format!(
7347                        "round {round}: votes split ({}) — one round of reconsideration",
7348                        initial_votes
7349                            .iter()
7350                            .map(|v| v.label())
7351                            .collect::<Vec<_>>()
7352                            .join(", ")
7353                    ),
7354                );
7355                // Seats read every seat's findings and votes, still numbered
7356                // and never named — the same anonymity `review` itself keeps.
7357                let panel: Vec<ReviewSeatReport<'_>> = records
7358                    .iter()
7359                    .filter_map(|r| {
7360                        r.vote.map(|vote| ReviewSeatReport {
7361                            reviewer: r.reviewer,
7362                            vote,
7363                            summary: &r.summary,
7364                            findings: &r.findings,
7365                        })
7366                    })
7367                    .collect();
7368
7369                let mut jobs = Vec::new();
7370                let mut seats_at = Vec::new();
7371                for (r, spec) in reviewers.iter().cloned().enumerate() {
7372                    // A seat with no initial vote has nothing to reconsider
7373                    // from and stays absent, the same as it stayed absent
7374                    // from `panel` above.
7375                    if records[r].vote.is_none() {
7376                        continue;
7377                    }
7378                    let wt = root.join(format!("review-{}", r + 1));
7379                    let seat_key = format!("review-{}", r + 1);
7380                    let spec = self.occupant(&seat_key, spec);
7381                    let seat = self.seat(&seat_key, &spec.id);
7382                    // A seat with no live session has already forgotten the
7383                    // initial review's prompt — restate the patch it is
7384                    // voting on, the same as `deliberate`/`vote` do for a
7385                    // judge in the same position.
7386                    // The panel already carries this seat's own review and
7387                    // vote, so restating the patch makes the prompt whole for
7388                    // a seat handed to another agent.
7389                    let build = |with_patch: bool| {
7390                        prompt::review_reconsider(&ReviewReconsiderCtx {
7391                            instruction: &self.state.instruction,
7392                            reviewer: r + 1,
7393                            lens: Lens::for_seat(r),
7394                            panel: &panel,
7395                            patch: with_patch.then_some(ReviewPatch {
7396                                branch: &winner.branch,
7397                                base_short: &base_short,
7398                                stat: &stat,
7399                                patch: &patch,
7400                            }),
7401                            round,
7402                            rounds: max_rounds,
7403                            language: &language,
7404                        })
7405                    };
7406                    let full = build(true);
7407                    let prompt = if has_context(&spec, &seat, sessions) {
7408                        build(false)
7409                    } else {
7410                        full.clone()
7411                    };
7412                    jobs.push(SeatJob {
7413                        prompt,
7414                        spec,
7415                        seat,
7416                        cwd: wt,
7417                        timeout: Duration::from_secs(self.state.config.graph.timeout_review),
7418                        allow_write: false,
7419                        sessions,
7420                        artifacts: artifacts.clone(),
7421                        stem: format!("review-{round}-reconsider-{}", r + 1),
7422                        handover: Some(full),
7423                    });
7424                    seats_at.push(r);
7425                }
7426
7427                let mut recon_quota_losses = Vec::new();
7428                let recon_cache = self.state.config.cache_dir();
7429                let recon_ctx = WaveCtx {
7430                    carry_seats: true,
7431                    pauses: Some((&self.pause, &self.interrupt)),
7432                    run: &run_id,
7433                    node: "review",
7434                    prompts: &prompts,
7435                    cache: recon_cache.as_deref(),
7436                    round: Some(round),
7437                };
7438                let recon_results = ask_json_wave::<ReviewRevote>(
7439                    jobs,
7440                    Arc::clone(&self.sem),
7441                    review_retries,
7442                    &self.roles.reviewer_roster,
7443                    &recon_ctx,
7444                    &mut recon_quota_losses,
7445                    &mut self.state,
7446                    &|_: &ReviewRevote| Ok(()),
7447                )
7448                .await;
7449                if self.state.resource_block.is_some() {
7450                    return self.stop_for_resource();
7451                }
7452                self.state.quota.extend(recon_quota_losses);
7453
7454                for (&r, (seat, res, _attempts)) in seats_at.iter().zip(recon_results) {
7455                    let agent_id = seat.agent.clone();
7456                    self.state.seats.insert(seat.key.clone(), seat);
7457                    let mut rec = ReviewRevoteRecord {
7458                        reviewer: r + 1,
7459                        agent: agent_id,
7460                        vote: None,
7461                        reason: String::new(),
7462                        failed: None,
7463                    };
7464                    match res {
7465                        Ok((rv, _)) => {
7466                            rec.vote = Some(rv.vote);
7467                            rec.reason =
7468                                blind::sanitize_prose(&rv.reason, &self.state.config.blind);
7469                            self.state.event(
7470                                "review",
7471                                format!(
7472                                    "round {round}: reviewer {} revoted {}",
7473                                    r + 1,
7474                                    rv.vote.label()
7475                                ),
7476                            );
7477                        }
7478                        Err(e) => {
7479                            rec.failed = Some(e.to_string());
7480                            self.state.event(
7481                                "review",
7482                                format!("round {round}: reviewer {} did not revote: {e}", r + 1),
7483                            );
7484                        }
7485                    }
7486                    reconsideration.push(rec);
7487                }
7488            } else if initial_votes.len() > 1 {
7489                self.state.event(
7490                    "review",
7491                    format!(
7492                        "round {round}: votes agreed ({}) — no reconsideration",
7493                        initial_votes[0].label()
7494                    ),
7495                );
7496            }
7497
7498            let blocking = all_findings.iter().filter(|f| f.severity.blocks()).count();
7499            let verify_timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
7500            // A round that already has a blocking finding and a round left to
7501            // try is going back to the fixer no matter what `verify.e2e`
7502            // says, so running it first only spends the loop's slowest step
7503            // (minutes, for a Rust repo's full test suite) on a head about
7504            // to be rewritten. Deferred, never skipped: `verify.e2e` still
7505            // runs once a round has no blocking findings left (see
7506            // `round_is_clean`, which a deferred — empty — `e2e` can never
7507            // satisfy since `blocking` is nonzero whenever this branch is
7508            // taken), and `stop_reviewing` forces a real run before it will
7509            // ever read a deferred round as green.
7510            let defer_e2e =
7511                blocking > 0 && round < max_rounds && !self.state.config.graph.e2e_every_round;
7512            let (e2e, verify_retried, e2e_deferred, e2e_defer_reason) = if defer_e2e {
7513                let reason =
7514                    format!("{blocking} blocking finding(s) already required a fix this round");
7515                self.state.event(
7516                    "verify",
7517                    format!(
7518                        "round {round}: {reason} — e2e deferred to the fixer (reviewed head \
7519                         {}); it will run once a round has none left",
7520                        short(&head)
7521                    ),
7522                );
7523                (Vec::new(), false, true, Some(reason))
7524            } else {
7525                let e2e_commands = self.state.config.verify.e2e.clone();
7526                let cache_dir = self.state.config.cache_dir();
7527                let context = format!("round {round}");
7528                let wait = self.repository_wait();
7529                let (e2e, verify_retried) = with_cache_lease(
7530                    &mut self.state,
7531                    cache_dir.as_deref(),
7532                    "e2e",
7533                    "e2e",
7534                    &winner.worktree,
7535                    &head,
7536                    verify_timeout,
7537                    &context,
7538                    &wait,
7539                    |state, budget| {
7540                        let shell = shell.clone();
7541                        let e2e_commands = e2e_commands.clone();
7542                        let worktree = winner.worktree.clone();
7543                        let context = context.clone();
7544                        async move {
7545                            run_e2e_with_retry(
7546                                state,
7547                                &shell,
7548                                &e2e_commands,
7549                                &worktree,
7550                                budget,
7551                                &context,
7552                            )
7553                            .await
7554                        }
7555                    },
7556                )
7557                .await?;
7558                (e2e, verify_retried, false, None)
7559            };
7560
7561            let expected = records.len();
7562            let answered = records.iter().filter(|r| r.failed.is_none()).count();
7563            let incomplete = answered < expected;
7564            let e2e_ok = e2e.iter().all(CommandOutcome::ok);
7565            let policy = self.state.config.graph.incomplete_review;
7566            let clean = round_is_clean(
7567                blocking,
7568                e2e_ok,
7569                answered,
7570                expected,
7571                round_quota_missing,
7572                policy,
7573            );
7574
7575            let mut round_record = ReviewRound {
7576                round,
7577                head: head.clone(),
7578                verified_head: None,
7579                verified_at: None,
7580                reviews: records,
7581                e2e,
7582                verify_retried,
7583                e2e_deferred,
7584                e2e_defer_reason,
7585                fix: None,
7586                blocking,
7587                answered,
7588                expected,
7589                clean,
7590                progressed: false,
7591                vote_split,
7592                reconsideration,
7593                verdict: None,
7594            };
7595            // The final vote per seat is its revote where reconsideration
7596            // ran and answered, its initial vote otherwise — the same
7597            // fallback `tally` uses for a judge whose private vote failed.
7598            round_record.verdict = ReviewVote::worst(
7599                round_record
7600                    .final_votes()
7601                    .into_iter()
7602                    .map(|(_, _, vote)| vote),
7603            );
7604            // Which commit and when magi actually attempted to check —
7605            // known the moment a command was dispatched against `head`,
7606            // whether or not it finished: a resource-blocked attempt still
7607            // targeted a specific commit at a specific time, and leaving
7608            // that unrecorded is exactly what made `verification_summary`
7609            // report a fresh attempt as "commit unknown ... recorded before
7610            // this was tracked", indistinguishable from a genuinely old,
7611            // untracked record. Only a deferred or unconfigured round never
7612            // ran at all and has nothing to record — see
7613            // `ReviewRound::verified_head`'s own doc.
7614            if !matches!(
7615                round_record.e2e_status(),
7616                E2eStatus::Deferred | E2eStatus::NotConfigured
7617            ) {
7618                round_record.verified_head = Some(head.clone());
7619                round_record.verified_at = Some(Timestamp::now());
7620            }
7621
7622            if incomplete {
7623                let missing: Vec<String> = round_record
7624                    .reviews
7625                    .iter()
7626                    .filter(|r| r.failed.is_some())
7627                    .map(|r| format!("review-{}", r.reviewer))
7628                    .collect();
7629                self.state.event(
7630                    "review",
7631                    format!(
7632                        "round {round}: {answered}/{expected} reviewer(s) answered ({} never answered)",
7633                        missing.join(", ")
7634                    ),
7635                );
7636            }
7637
7638            if clean {
7639                self.state.event(
7640                    "review",
7641                    if incomplete && policy == IncompleteReviewPolicy::Warn {
7642                        format!(
7643                            "round {round}: clean (warn policy, incomplete panel) — no \
7644                             blocking findings from the seats that answered, verification green"
7645                        )
7646                    } else if incomplete {
7647                        format!(
7648                            "round {round}: clean ({} rate-limited reviewer(s) excluded from \
7649                             quorum) — no blocking findings from the seats that answered, \
7650                             verification green",
7651                            expected - answered
7652                        )
7653                    } else {
7654                        format!("round {round}: clean — no blocking findings, verification green")
7655                    },
7656                );
7657                self.state.reviews.push(round_record);
7658                self.state.status = RunStatus::Gating;
7659                self.state.save()?;
7660                return Ok(());
7661            }
7662
7663            // Nothing was raised and verification passed, but not every seat
7664            // answered and `round_is_clean` still refused to call it clean —
7665            // either a seat is missing for a reason other than its own quota
7666            // (a crash, a timeout, unparsable output — worth another try), or
7667            // every seat that could have answered lost its quota and nobody
7668            // is left to decide on: re-review rather than send the fixer
7669            // after a round with nothing to fix.
7670            if incomplete && blocking == 0 && e2e_ok {
7671                self.state.reviews.push(round_record);
7672                self.state.save()?;
7673                if round == max_rounds {
7674                    self.state.status = RunStatus::Blocked;
7675                    self.state.event(
7676                        "review",
7677                        format!(
7678                            "{} reviewer seat(s) never answered after {max_rounds} rounds; \
7679                             refusing to call it clean",
7680                            expected - answered
7681                        ),
7682                    );
7683                    return Ok(());
7684                }
7685                continue;
7686            }
7687
7688            // Nothing for the fixer to act on (`blocking == 0`) and the only
7689            // reason this round is not clean is that magi itself never got
7690            // a command to run — the shared build cache, not the patch (see
7691            // `CommandOutcome::resource_blocked`'s own doc). Sending that to
7692            // the fixer would invite a change to appease contention that has
7693            // nothing to do with the diff, and would leave this attempt
7694            // sitting in the next round's prompt as if it were about an
7695            // earlier, superseded commit rather than what it actually is:
7696            // the same head, still waiting to be checked. Wait for it the
7697            // same way the final round's own contention is already handled,
7698            // whatever round this happens to be.
7699            if blocking == 0 && round_record.e2e_status() == E2eStatus::ResourceBlocked {
7700                self.state.reviews.push(round_record);
7701                return self
7702                    .stop_unresolved_review(
7703                        "the round's own verification could not run",
7704                        &shell,
7705                        &winner.worktree,
7706                    )
7707                    .await;
7708            }
7709
7710            if round == max_rounds {
7711                self.state.reviews.push(round_record);
7712                return self
7713                    .stop_reviewing(
7714                        &format!(
7715                            "{blocking} blocking finding(s) still open after {max_rounds} round(s)"
7716                        ),
7717                        &shell,
7718                        &winner.worktree,
7719                    )
7720                    .await;
7721            }
7722
7723            if !self.fix_review_round(&winner, round_record).await? {
7724                return Ok(());
7725            }
7726        }
7727        Ok(())
7728    }
7729
7730    /// Run the same fix chain for an in-loop failure or a persisted catch-up failure.
7731    /// False means admission or continuation policy stopped the review loop.
7732    async fn fix_review_round(
7733        &mut self,
7734        winner: &Candidate,
7735        mut round_record: ReviewRound,
7736    ) -> Result<bool> {
7737        let round = round_record.round;
7738        let shell = self.state.config.shell();
7739        let step = self.run_fix_round(winner, &mut round_record).await?;
7740        if matches!(step, FixStep::Resource) {
7741            self.stop_for_resource()?;
7742            return Ok(false);
7743        }
7744        if self.state.reviews.len() < round {
7745            self.state.reviews.push(round_record);
7746        } else {
7747            self.state.reviews[round - 1] = round_record;
7748        }
7749        self.state.save()?;
7750
7751        // The fixer's own report never came back this round, even after
7752        // `continue_fix_report`'s own budget was spent on it — not an
7753        // ordinary "no report" (dropped stream, quota, plain failure),
7754        // which already reads that way and is left to the existing round
7755        // budget. Stopping here, rather than opening another round, is
7756        // what keeps a next reviewer/fixer wave from ever being
7757        // dispatched onto `winner.worktree` while whatever the seat's
7758        // last call may still have running there is unaccounted for: no
7759        // process liveness check exists (and none is being added — see
7760        // AGENTS.md/this task's own scope), so the only way to honour
7761        // "nothing starts before a valid report returns" is to not start
7762        // anything further on this worktree from this run at all.
7763        if matches!(step, FixStep::Recorded { report_lost: true }) {
7764            self.stop_reviewing(
7765                "the fixer's adoption report never came back, even after resuming its \
7766                         own seat; refusing to start another round against the same worktree \
7767                         while that is unresolved",
7768                &shell,
7769                &winner.worktree,
7770            )
7771            .await?;
7772            return Ok(false);
7773        }
7774
7775        let streak = self
7776            .state
7777            .reviews
7778            .iter()
7779            .rev()
7780            .take_while(|r| !r.progressed)
7781            .count();
7782        if streak >= STAGNANT_LIMIT {
7783            self.stop_reviewing(
7784                &format!("the tree has not moved against base for {streak} round(s) in a row"),
7785                &shell,
7786                &winner.worktree,
7787            )
7788            .await?;
7789            return Ok(false);
7790        }
7791        Ok(true)
7792    }
7793
7794    /// Execute the shared fixer chain; the caller owns persistence and stop policy.
7795    async fn run_fix_round(
7796        &mut self,
7797        winner: &Candidate,
7798        round_record: &mut ReviewRound,
7799    ) -> Result<FixStep> {
7800        let round = round_record.round;
7801        let max_rounds = self.state.config.graph.review_rounds;
7802        let run_id = self.state.id.clone();
7803        let prompts = self.state.config.prompts.clone();
7804        let language = self.state.config.graph.language.clone();
7805        let sessions = self.state.config.graph.sessions;
7806        let artifacts = self.merge_repair_artifacts();
7807        let base = self.landing_base();
7808        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
7809        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
7810        let this_round_verification = round_record.verification_summary(&head);
7811        let all_findings: Vec<_> = round_record
7812            .reviews
7813            .iter()
7814            .flat_map(|r| r.findings.iter().cloned())
7815            .collect();
7816        // Fix. The winner's own implementer seat continues its conversation:
7817        // the competition is over, so context is pure benefit now.
7818        let blocking_findings: Vec<_> = all_findings
7819            .iter()
7820            .filter(|f| f.severity.blocks())
7821            .cloned()
7822            .collect();
7823        let fix_prompt = prompt::fix_in(
7824            &self.state.instruction,
7825            &blocking_findings,
7826            this_round_verification.as_ref(),
7827            round,
7828            max_rounds,
7829            &language,
7830            &self.state.config.graph.github_language,
7831        );
7832        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
7833        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
7834        let (job, seat, out) = self
7835            .ask_fixer(winner, "fix", Some(round), |spec, seat| SeatJob {
7836                prompt: fix_prompt.clone(),
7837                spec,
7838                seat,
7839                cwd: winner.worktree.clone(),
7840                timeout,
7841                allow_write: true,
7842                sessions,
7843                artifacts: artifacts.clone(),
7844                stem: format!("fix-{round}"),
7845                handover: None,
7846            })
7847            .await;
7848        // Not asked: the round is not recorded (so a resume repeats it),
7849        // no fixer was handed over to, and the seat is left as it was.
7850        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
7851            return Ok(FixStep::Resource);
7852        }
7853        let agent_id = seat.agent.clone();
7854
7855        let mut fix = FixRecord {
7856            agent: agent_id,
7857            addressed: Vec::new(),
7858            rejected: Vec::new(),
7859            notes: String::new(),
7860            committed: false,
7861            failed: None,
7862            duration_ms: 0,
7863            continuation: None,
7864        };
7865        let mut continuation = ContinuationRecord::not_needed();
7866        let mut final_seat = seat.clone();
7867        match out {
7868            AgentOutcome::Ok(o) => {
7869                fix.duration_ms = o.duration_ms;
7870                let parsed = verdict::extract_json::<FixReport>(&o.text);
7871                // A parsed report standing next to a command this same
7872                // reply's own CLI never confirmed the exit status of is
7873                // not a resolved answer — the identical `CommandEvidence`
7874                // `state.jobs` renders, read here instead of only on
7875                // display, per the completion judgment and the shown
7876                // record needing to agree.
7877                let incomplete_reason = match &parsed {
7878                    Ok(_) if has_unconfirmed_command(&o.commands) => Some(
7879                        "the reply parsed, but it reported a command whose own CLI never \
7880                             confirmed an exit status"
7881                            .to_owned(),
7882                    ),
7883                    Ok(_) => None,
7884                    Err(e) => Some(e.to_string()),
7885                };
7886                match incomplete_reason {
7887                    None => {
7888                        let report = parsed.expect("checked Ok above");
7889                        fix.addressed = report.addressed;
7890                        fix.rejected = report.rejected;
7891                        fix.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
7892                    }
7893                    Some(reason) => {
7894                        let (resumed_seat, resolved, failure, cont) = self
7895                            .continue_fix_report(seat, reason, &job, &prompts, &run_id, round)
7896                            .await;
7897                        fix.duration_ms += cont.cumulative_wait_ms;
7898                        continuation = cont;
7899                        final_seat = resumed_seat;
7900                        match resolved {
7901                            Some(report) => {
7902                                fix.addressed = report.addressed;
7903                                fix.rejected = report.rejected;
7904                                fix.notes =
7905                                    blind::sanitize_prose(&report.notes, &self.state.config.blind);
7906                            }
7907                            None => fix.failed = failure,
7908                        }
7909                    }
7910                }
7911            }
7912            // The CLI's raw error JSON is not a fix report to parse.
7913            AgentOutcome::Dropped(o) => {
7914                fix.duration_ms = o.duration_ms;
7915                let why = o
7916                    .dropped
7917                    .as_ref()
7918                    .map(|d| d.why.as_str())
7919                    .unwrap_or("the CLI ended the stream without delivering its answer");
7920                fix.failed = Some(format!("the CLI dropped the stream ({why})"));
7921            }
7922            AgentOutcome::Quota(o) => {
7923                self.state.quota.push(QuotaLoss::from_report(
7924                    final_seat.key.clone(),
7925                    "fix",
7926                    o.quota.as_ref(),
7927                ));
7928                fix.failed = Some("rate limited (quota); fixer could not run".to_owned());
7929            }
7930            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => fix.failed = Some(e),
7931        }
7932        if self.state.resource_block.is_some() {
7933            self.state.seats.insert(final_seat.key.clone(), final_seat);
7934            return Ok(FixStep::Resource);
7935        }
7936        fix.continuation = Some(continuation);
7937        self.state.seats.insert(final_seat.key.clone(), final_seat);
7938        let coordination = self.coordinate(&winner.worktree).await?;
7939        if let Ok(r) = git::rescue_commit(
7940            &winner.worktree,
7941            &format!("magi: review round {round} fixes (uncommitted work)"),
7942        )
7943        .await
7944        {
7945            self.state.note_withheld("fix", &r.withheld);
7946        }
7947        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
7948        fix.committed = after != before;
7949        // Judged by what `git` says moved against base, never by the
7950        // fixer's own `addressed`/`rejected` count — see
7951        // `ReviewRound::progressed`. Propagated with `?`, the same as the
7952        // `patch` snapshot above: swallowing this error would default
7953        // `diff_after` to empty, which almost always differs from a
7954        // non-empty `patch` and reads as "progressed" — exactly backwards
7955        // for a `git` failure the stagnation check cannot see through.
7956        let diff_after = git::diff(&winner.worktree, &base, "HEAD").await?;
7957        drop(coordination);
7958        let progressed = diff_after != patch;
7959        let commit_note = if fix.committed {
7960            "committed"
7961        } else {
7962            "NO new commit"
7963        };
7964        let tree_note = if progressed {
7965            "changed vs base"
7966        } else {
7967            "unchanged vs base"
7968        };
7969        self.state.event(
7970            "fix",
7971            match &fix.failed {
7972                // Distinct on purpose from "0 addressed, 0 rejected": the
7973                // fixer's own diff still landed (blocking counts do keep
7974                // falling round over round), only its adoption report did
7975                // not come back, so this must never read like every
7976                // finding was reviewed and declined.
7977                Some(reason) => {
7978                    format!(
7979                        "round {round}: fixer's adoption report was lost ({reason}); \
7980                             {commit_note}, tree {tree_note}"
7981                    )
7982                }
7983                None => format!(
7984                    "round {round}: {} addressed, {} rejected, {commit_note}, tree \
7985                         {tree_note}{}",
7986                    fix.addressed.len(),
7987                    fix.rejected.len(),
7988                    if continuation.outcome == ContinuationOutcome::Resumed {
7989                        format!(
7990                            " (adoption report recovered after {} continuation(s))",
7991                            continuation.attempts
7992                        )
7993                    } else {
7994                        String::new()
7995                    },
7996                ),
7997            },
7998        );
7999        round_record.fix = Some(fix);
8000        round_record.progressed = progressed;
8001        Ok(FixStep::Recorded {
8002            report_lost: matches!(
8003                continuation.outcome,
8004                ContinuationOutcome::Exhausted
8005                    | ContinuationOutcome::QuotaLost
8006                    | ContinuationOutcome::NoSession
8007            ),
8008        })
8009    }
8010
8011    /// Decide, from the last recorded round's own verification, whether
8012    /// stopping the review loop is a hand-off or a genuine block.
8013    ///
8014    /// Called once the loop has given up trying — the round budget is spent,
8015    /// or the tree stopped moving (see [`STAGNANT_LIMIT`]) — with blocking
8016    /// findings still open, never while a round is still clean or the
8017    /// incomplete-panel case handled inline above. Gate and e2e are facts
8018    /// about the tree; a lingering review finding is an opinion, and this
8019    /// workload's own `magi stats` puts reviewer precision low enough
8020    /// (12-33%, 0.18-0.29 adopted per round) that a panel of open findings
8021    /// must not by itself stand between a green, verified change and the
8022    /// human who decides what to do with it. A red e2e is not an opinion, so
8023    /// that case still blocks, with the failing command and a tail of its
8024    /// output recorded here rather than left in `run.json` for someone to go
8025    /// find.
8026    ///
8027    /// A round that deferred its own e2e (see [`Config::graph`]'s
8028    /// `e2e_every_round`) is never read as that green: its `e2e` is empty
8029    /// only because nothing ran, and treating an empty list as a passing one
8030    /// here is exactly the "deferred painted green" bug this function exists
8031    /// to not have. When the last round's own verification never resolved —
8032    /// deferred on purpose, or a real attempt the shared build cache blocked
8033    /// — this makes (or retries) the real run, on the actual worktree this
8034    /// loop is about to stop touching, before deciding anything. A
8035    /// resource-blocked attempt is likewise never read as either green or
8036    /// red: it is evidence about the machine, not the patch (see
8037    /// [`CommandOutcome::resource_blocked`]'s own doc), so a persistently
8038    /// blocked cache leaves this call without deciding rather than guessing
8039    /// — the caller retries on a later reentry.
8040    /// Record, once, that the review loop handed off over a blocking finding
8041    /// a reviewer rejected on (see [`ReviewRound::contested_handoff`]), so
8042    /// `land` asks the owner even with `land_approval` off. Called from every
8043    /// path that concludes `Gating`; a reentry keeps the first record.
8044    fn record_contested_handoff(&mut self) {
8045        if self.state.contested_handoff.is_some() {
8046            return;
8047        }
8048        let Some(contested) = self
8049            .state
8050            .reviews
8051            .last()
8052            .and_then(ReviewRound::contested_handoff)
8053        else {
8054            return;
8055        };
8056        self.state.event(
8057            "review",
8058            format!(
8059                "{} blocking finding(s) open and {} reviewer(s) rejecting — the merge will \
8060                 wait for the owner's approval",
8061                contested.findings.len(),
8062                contested.rejecters.len()
8063            ),
8064        );
8065        self.state.contested_handoff = Some(contested);
8066    }
8067
8068    async fn stop_unresolved_review(
8069        &mut self,
8070        why: &str,
8071        shell: &[String],
8072        worktree: &Path,
8073    ) -> Result<()> {
8074        self.stop_reviewing_inner(why, shell, worktree, true).await
8075    }
8076
8077    async fn stop_reviewing(&mut self, why: &str, shell: &[String], worktree: &Path) -> Result<()> {
8078        self.stop_reviewing_inner(why, shell, worktree, false).await
8079    }
8080
8081    async fn stop_reviewing_inner(
8082        &mut self,
8083        why: &str,
8084        shell: &[String],
8085        worktree: &Path,
8086        repairable: bool,
8087    ) -> Result<()> {
8088        let round_idx = self.state.reviews.len() - 1;
8089        // A deferred round and a resource-blocked one are the same shape
8090        // here: neither has a real result yet, and both get one more
8091        // attempt. Read off `e2e_status` — the single source for this —
8092        // rather than `e2e.is_empty()` alone, so a resource-blocked attempt
8093        // (whose `e2e` is *not* empty; see `CommandOutcome::resource_blocked`)
8094        // still retries instead of being read as a settled result the
8095        // instant it stops being empty.
8096        let needs_catchup_run = matches!(
8097            self.state.reviews[round_idx].e2e_status(),
8098            E2eStatus::Deferred | E2eStatus::ResourceBlocked
8099        );
8100        if needs_catchup_run {
8101            let round = self.state.reviews[round_idx].round;
8102            let timeout = Duration::from_secs(self.state.config.graph.verify_timeout());
8103            let commands = self.state.config.verify.e2e.clone();
8104            let attempted_head = git::rev_parse(worktree, "HEAD").await?;
8105            let cache_dir = self.state.config.cache_dir();
8106            let context = format!(
8107                "round {round}: verification unresolved, catching up before the final decision"
8108            );
8109            let wait = self.repository_wait();
8110            let (outcomes, verify_retried) = with_cache_lease(
8111                &mut self.state,
8112                cache_dir.as_deref(),
8113                "e2e",
8114                "e2e",
8115                worktree,
8116                &attempted_head,
8117                timeout,
8118                &context,
8119                &wait,
8120                |state, budget| {
8121                    let shell = shell.to_vec();
8122                    let commands = commands.clone();
8123                    let context = context.clone();
8124                    async move {
8125                        run_e2e_with_retry(state, &shell, &commands, worktree, budget, &context)
8126                            .await
8127                    }
8128                },
8129            )
8130            .await?;
8131            if !outcomes.is_empty() && !verify_inconclusive(&outcomes) {
8132                self.state.record_base_progress(&attempted_head);
8133            }
8134            let last = &mut self.state.reviews[round_idx];
8135            last.e2e = outcomes;
8136            last.verify_retried = verify_retried;
8137            // Always the commit and time this attempt actually targeted,
8138            // whether or not it happens to equal the reviewed `head` and
8139            // whether or not a command finished — see
8140            // `ReviewRound::verified_head`'s own doc. A still-inconclusive
8141            // attempt is recorded too, so a later reader sees "attempted
8142            // again at T2" rather than silence.
8143            last.verified_head = Some(attempted_head);
8144            last.verified_at = Some(Timestamp::now());
8145            if verify_inconclusive(&last.e2e) {
8146                // Still not a real result: `e2e_deferred` is left exactly
8147                // as it was, so `needs_catchup_run` above reads
8148                // `ResourceBlocked` (via `e2e_status`, which checks
8149                // `resource_blocked` before `e2e_deferred`) and retries
8150                // again on the next reentry, rather than recording
8151                // contention as a red e2e and blocking the run on it.
8152                self.state.save()?;
8153                return Ok(());
8154            }
8155            last.e2e_deferred = false;
8156            // A complete panel may have been saved as unclean solely because
8157            // verification could not run. Refresh that verdict with the new
8158            // outcomes before a granted repair consumes the round's result.
8159            // Incomplete panels retain their original policy/quota decision.
8160            if last.answered == last.expected {
8161                last.clean = round_is_clean(
8162                    last.blocking,
8163                    last.e2e.iter().all(CommandOutcome::ok),
8164                    last.answered,
8165                    last.expected,
8166                    0,
8167                    self.state.config.graph.incomplete_review,
8168                );
8169            }
8170        }
8171        let last = &self.state.reviews[round_idx];
8172        let open: usize = last.reviews.iter().map(|r| r.findings.len()).sum();
8173
8174        match last.e2e_status() {
8175            E2eStatus::Failed => {
8176                let red: Vec<String> = last
8177                    .e2e
8178                    .iter()
8179                    .filter(|o| !o.ok())
8180                    .map(|o| {
8181                        format!(
8182                            "`{}` -> {:?}\n{}",
8183                            o.command,
8184                            o.code,
8185                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8186                        )
8187                    })
8188                    .collect();
8189                let can_repair = last.blocking == 0 && last.fix.is_none();
8190                self.state.reviews[round_idx].clean = false;
8191                self.state
8192                    .event("review", format!("{why}; e2e failed:\n{}", red.join("\n")));
8193                self.state.status = RunStatus::Blocked;
8194                if repairable
8195                    && can_repair
8196                    && self.state.reviews.len() < self.state.config.graph.review_rounds
8197                {
8198                    self.state.status = RunStatus::Reviewing;
8199                }
8200            }
8201            // `needs_catchup_run` above already retried once this call; if
8202            // it is still blocked, this is magi's own admission it could
8203            // not get a command to run, never a verdict on the patch — the
8204            // run is left exactly where a later reentry can retry again.
8205            E2eStatus::ResourceBlocked => {
8206                self.state.event(
8207                    "review",
8208                    format!(
8209                        "{why}; e2e could not run (resource unavailable); not \
8210                         deciding yet"
8211                    ),
8212                );
8213            }
8214            E2eStatus::Passed | E2eStatus::Deferred | E2eStatus::NotConfigured => {
8215                self.state.event(
8216                    "review",
8217                    format!("{why}; e2e is green — handing off with {open} finding(s) still open"),
8218                );
8219                self.record_contested_handoff();
8220                self.state.status = RunStatus::Gating;
8221            }
8222        }
8223        self.state.save()?;
8224        Ok(())
8225    }
8226
8227    // ----------------------------------------------------------------- gate
8228
8229    async fn gate(&mut self) -> Result<()> {
8230        if self.hold_base_sync().await? {
8231            return Ok(());
8232        }
8233        // Judged by the review record itself, not by `status`: a solo
8234        // candidate's `judge`/`deliberate` skip rewrites `status` on every
8235        // reentry (see `judge`), and trusting it here is exactly how a run
8236        // that exhausted its review budget got gated and merged a second
8237        // time around. `review_conclusion` recomputes the review loop's own
8238        // verdict from the round records themselves — `Gating` for a clean
8239        // round or a hand-off (see `stop_reviewing`), anything else means the
8240        // loop is still going or genuinely blocked.
8241        // A base the winner could not be replayed onto is a decision, not a
8242        // round: there is no landing tree to gate. Read as its own record for
8243        // the same reason the review verdict is.
8244        if self.state.status == RunStatus::Failed
8245            || self
8246                .state
8247                .base_sync
8248                .as_ref()
8249                .is_some_and(|s| s.conflict.is_some())
8250            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
8251                != Some(RunStatus::Gating)
8252        {
8253            return Ok(());
8254        }
8255        let Some(winner) = self.state.winner().cloned() else {
8256            return Ok(());
8257        };
8258        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
8259        self.state.reconcile_verification_head(&head);
8260        self.state.save()?;
8261        let wait = self.repository_wait();
8262        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8263            return Ok(());
8264        }
8265        if self.state.gate_ran {
8266            // `review_loop` derives its conclusion from the clean review
8267            // record on every reentry and therefore puts a completed run back
8268            // in `Gating`. A recorded gate is a stronger, terminal fact:
8269            // retain its original command output (or lack of any, for a repo
8270            // with no `verify.gate` commands — see `RunState::gate_ran`'s own
8271            // doc) and restore `Blocked` on a real failure rather than
8272            // pretending the command is still running or running it a second
8273            // time. `gate_ran == false` remains the only shape — unattempted,
8274            // or a resource-blocked retry — that may still need to execute a
8275            // command.
8276            if self.state.gate.iter().any(|outcome| !outcome.ok()) {
8277                self.state.status = RunStatus::Blocked;
8278                self.state.save()?;
8279            }
8280            return Ok(());
8281        }
8282        self.state.status = RunStatus::Gating;
8283        let mut outcomes = self.run_gate(&winner).await?;
8284        let mut attempted_head = self.state.gate_head.clone();
8285        loop {
8286            // A resource-blocked outcome means the gate command never actually
8287            // ran - the shared build cache could not be acquired or confirmed
8288            // fresh in time - which is evidence about the machine, not about
8289            // the tree (see `CommandOutcome::resource_blocked`'s own doc).
8290            // Recording it as a red gate would mark a run `Blocked` on nothing
8291            // but contention magi has already logged; leaving `self.state.gate`
8292            // empty and `self.state.gate_ran` false instead keeps the shape
8293            // this function already treats as "still needs to run" (see the
8294            // early-return above), so the next call retries the command
8295            // rather than concluding anything.
8296            if verify_inconclusive(&outcomes) {
8297                self.state.save()?;
8298                return Ok(());
8299            }
8300            if outcomes.iter().all(CommandOutcome::ok) {
8301                break;
8302            }
8303            let fixes_before = self.state.gate_fixes.len();
8304            match self.gate_fix_round(&winner, &outcomes).await? {
8305                GateFix::Retry => {
8306                    outcomes = self.run_gate(&winner).await?;
8307                    attempted_head = self.state.gate_head.clone();
8308                }
8309                GateFix::Stop => {
8310                    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
8311                    if self.state.gate_fixes.len() > fixes_before
8312                        && attempted_head.as_deref() != Some(current.as_str())
8313                        && verify_current_head(&mut self.state, &winner, &wait).await?
8314                    {
8315                        // A new SHA owes a gate even when a spent fixer changed only
8316                        // commit metadata. This does not grant another repair round.
8317                        outcomes = self.run_gate(&winner).await?;
8318                        attempted_head = self.state.gate_head.clone();
8319                    } else {
8320                        break;
8321                    }
8322                }
8323                GateFix::Defer => {
8324                    self.state.save()?;
8325                    return Ok(());
8326                }
8327            }
8328        }
8329        let passed = outcomes.iter().all(CommandOutcome::ok);
8330        self.state.gate = outcomes;
8331        self.state.gate_ran = true;
8332        self.state.gate_head = attempted_head;
8333        if !self.state.gate.is_empty()
8334            && !verify_inconclusive(&self.state.gate)
8335            && let Some(head) = self.state.gate_head.clone()
8336        {
8337            self.state.record_base_progress(&head);
8338        }
8339        self.state.save()?;
8340        if !verify_current_head(&mut self.state, &winner, &wait).await? {
8341            return Ok(());
8342        }
8343        if !passed {
8344            self.state.status = RunStatus::Blocked;
8345            let spent = self.state.gate_fixes.len();
8346            self.state.event(
8347                "gate",
8348                if spent == 0 {
8349                    "gate failed; not merging".to_owned()
8350                } else {
8351                    format!("gate failed after {spent} gate-fix round(s); not merging")
8352                },
8353            );
8354        }
8355        self.state.save()?;
8356        Ok(())
8357    }
8358
8359    /// Run `verify.pre_gate` in the winner's worktree, then fold whatever it
8360    /// changed into one commit. Reached only from [`Self::run_gate`], i.e.
8361    /// after review is clean and never on a candidate awaiting judging.
8362    ///
8363    /// Never fails the run: a non-zero exit or timeout is a warning and a
8364    /// recorded outcome, and the gate remains the single arbiter. Nothing
8365    /// configured means nothing happens - no event, no commit. `commit_all`
8366    /// commits any leftover change under the neutral identity and returns
8367    /// `false` when the tree is clean, so no empty commit is ever made.
8368    async fn run_pre_gate(state: &mut RunState, winner: &Candidate) {
8369        let commands = state.config.verify.pre_gate.clone();
8370        if commands.is_empty() {
8371            return;
8372        }
8373        let shell = state.config.shell();
8374        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
8375        let (outcomes, _) = run_commands(
8376            state,
8377            "pre_gate",
8378            "pre_gate",
8379            0,
8380            &shell,
8381            &commands,
8382            &winner.worktree,
8383            timeout,
8384        )
8385        .await;
8386        for o in &outcomes {
8387            if !o.ok() {
8388                tracing::warn!(
8389                    "pre_gate `{}` failed ({:?}); the gate decides",
8390                    o.command,
8391                    o.code
8392                );
8393            }
8394            state.event(
8395                "pre_gate",
8396                format!(
8397                    "`{}` -> {}",
8398                    o.command,
8399                    if o.ok() {
8400                        "pass".to_owned()
8401                    } else {
8402                        format!(
8403                            "FAIL ({:?})\n{}",
8404                            o.code,
8405                            summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
8406                        )
8407                    }
8408                ),
8409            );
8410        }
8411        state.pre_gate = outcomes;
8412        match git::commit_all(&winner.worktree, "magi: pre_gate (mechanical fixes)").await {
8413            Ok(true) => match git::rev_parse(&winner.worktree, "HEAD").await {
8414                Ok(head) => {
8415                    state.event("pre_gate", format!("committed mechanical fixes ({head})"));
8416                    state.pre_gate_commit = Some(head);
8417                }
8418                Err(e) => tracing::warn!("pre_gate committed but HEAD unreadable: {e:#}"),
8419            },
8420            Ok(false) => {}
8421            Err(e) => tracing::warn!("pre_gate could not commit its changes: {e:#}"),
8422        }
8423        if let Err(e) = state.save() {
8424            tracing::warn!("could not persist the pre_gate record: {e:#}");
8425        }
8426    }
8427
8428    /// Run `verify.gate` once against the winner's current tree, logging one
8429    /// event per command. Empty when nothing is configured.
8430    async fn run_gate(&mut self, winner: &Candidate) -> Result<Vec<CommandOutcome>> {
8431        let wait = self.repository_wait();
8432        run_landing_gate(&mut self.state, winner, &wait).await
8433    }
8434
8435    /// One bounded fix round for a failing gate.
8436    ///
8437    /// The fixer is told the failure came from the gate itself, not from a
8438    /// reviewer, and is shown the failed commands, their exit codes and a tail
8439    /// of their output - whatever `[verify].gate` holds, nothing here knows
8440    /// what those commands run. Only a normal non-zero exit that printed
8441    /// something earns a round (see [`gate_fixable`]): a timeout, a missing
8442    /// command or a full disk says nothing about the code, and a fixer sent
8443    /// after it can only appease the machine. The round is judged by what git
8444    /// says moved, never by the fixer's own report, and `verify.e2e` runs
8445    /// again before the gate does, so a fix cannot trade a green gate for a
8446    /// red e2e unnoticed.
8447    async fn gate_fix_round(
8448        &mut self,
8449        winner: &Candidate,
8450        outcomes: &[CommandOutcome],
8451    ) -> Result<GateFix> {
8452        let cap = if self.state.repair_grants.last().is_some_and(|g| {
8453            g.phase == crate::repair_budget::Phase::Fixing
8454                && g.receipt.request.kind == crate::repair_budget::Kind::Gate
8455        }) {
8456            self.state.gate_fixes.len().saturating_add(1)
8457        } else {
8458            self.state.config.graph.gate_fix_rounds
8459        };
8460        let spent = self.state.gate_fixes.len();
8461        if spent >= cap {
8462            if cap > 0 {
8463                self.state.event(
8464                    "gate",
8465                    format!("{spent} gate-fix round(s) spent and the gate still fails"),
8466                );
8467            }
8468            return Ok(GateFix::Stop);
8469        }
8470        if !gate_fixable(outcomes) {
8471            self.state.event(
8472                "gate",
8473                "gate failure is not an ordinary non-zero exit with output (timeout, missing \
8474                 command or similar); not spending a fix round on it",
8475            );
8476            return Ok(GateFix::Stop);
8477        }
8478        let min_free = self.state.config.disk.min_free_bytes;
8479        if min_free > 0 {
8480            match crate::disk::free_bytes(&winner.worktree) {
8481                Ok(free) if crate::disk::enough_space(free, min_free) => {}
8482                Ok(free) => {
8483                    self.state.event(
8484                        "gate",
8485                        format!(
8486                            "only {free} bytes free ({min_free} required by `[disk] \
8487                             min_free_bytes`); not spending a fix round on a failure the disk \
8488                             may explain"
8489                        ),
8490                    );
8491                    return Ok(GateFix::Stop);
8492                }
8493                Err(e) => {
8494                    self.state.event(
8495                        "gate",
8496                        format!("free disk space could not be measured ({e:#}); no fix round"),
8497                    );
8498                    return Ok(GateFix::Stop);
8499                }
8500            }
8501        }
8502
8503        let attempt = spent + 1;
8504        let failed: Vec<CommandOutcome> = outcomes.iter().filter(|o| !o.ok()).cloned().collect();
8505        let base = self.landing_base();
8506        let fix_prompt = prompt::gate_fix_in(
8507            &self.state.instruction,
8508            &failed,
8509            attempt,
8510            cap,
8511            &self.state.config.graph.language,
8512            &self.state.config.graph.github_language,
8513        );
8514        let timeout = Duration::from_secs(self.state.config.graph.timeout_fix);
8515        let sessions = self.state.config.graph.sessions;
8516        let artifacts = self.merge_repair_artifacts();
8517        self.state.event(
8518            "gate",
8519            format!("gate failed; gate-fix round {attempt} of {cap}"),
8520        );
8521        let before = git::rev_parse(&winner.worktree, "HEAD").await?;
8522        let patch = git::diff(&winner.worktree, &base, "HEAD").await?;
8523        // Persist before invoking the fixer: it may commit and then be
8524        // interrupted before its result (or the following e2e) is saved.
8525        self.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
8526            head: before.clone(),
8527            passed: false,
8528            outcomes: Vec::new(),
8529        });
8530        self.state.save()?;
8531        let (_, seat, out) = self
8532            .ask_fixer(winner, "gate-fix", None, |spec, seat| SeatJob {
8533                prompt: fix_prompt.clone(),
8534                spec,
8535                seat,
8536                cwd: winner.worktree.clone(),
8537                timeout,
8538                allow_write: true,
8539                sessions,
8540                artifacts: artifacts.clone(),
8541                stem: format!("gate-fix-{attempt}"),
8542                handover: None,
8543            })
8544            .await;
8545        // The fixer was never asked: no round is spent (nothing is recorded)
8546        // and a later reentry asks the same agent again.
8547        if matches!(out, AgentOutcome::ResourceBlocked(_)) {
8548            return Ok(GateFix::Defer);
8549        }
8550        let mut record = GateFixRecord {
8551            agent: seat.agent.clone(),
8552            failed,
8553            notes: String::new(),
8554            committed: false,
8555            error: None,
8556        };
8557        match out {
8558            AgentOutcome::Ok(o) => {
8559                // A missing report is not a failed fix: the round is judged
8560                // by the tree below, and the report only carries prose.
8561                if let Ok(report) = verdict::extract_json::<FixReport>(&o.text) {
8562                    record.notes = blind::sanitize_prose(&report.notes, &self.state.config.blind);
8563                }
8564            }
8565            AgentOutcome::Dropped(_) => {
8566                record.error = Some("the CLI dropped the stream".to_owned());
8567            }
8568            AgentOutcome::Quota(o) => {
8569                self.state.quota.push(QuotaLoss::from_report(
8570                    seat.key.clone(),
8571                    "gate-fix",
8572                    o.quota.as_ref(),
8573                ));
8574                record.error = Some("rate limited (quota); fixer could not run".to_owned());
8575            }
8576            AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => record.error = Some(e),
8577        }
8578        self.state.seats.insert(seat.key.clone(), seat);
8579        let coordination = self.coordinate(&winner.worktree).await?;
8580        if let Ok(r) = git::rescue_commit(
8581            &winner.worktree,
8582            &format!("magi: gate fix {attempt} (uncommitted work)"),
8583        )
8584        .await
8585        {
8586            self.state.note_withheld("gate-fix", &r.withheld);
8587        }
8588        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
8589        record.committed = after != before;
8590        let changed = git::diff(&winner.worktree, &base, "HEAD").await? != patch;
8591        let note = record.error.clone();
8592        self.state.gate_fixes.push(record);
8593        self.state.save()?;
8594        if !changed {
8595            self.state.event(
8596                "gate",
8597                match note {
8598                    Some(why) => format!("gate-fix round {attempt}: fixer failed ({why})"),
8599                    None => format!("gate-fix round {attempt}: the tree did not change"),
8600                },
8601            );
8602            return Ok(GateFix::Stop);
8603        }
8604        self.state.event(
8605            "gate",
8606            format!("gate-fix round {attempt}: tree changed vs base; re-running verify.e2e"),
8607        );
8608
8609        drop(coordination);
8610        let wait = self.repository_wait();
8611        check_gate_repair_e2e(&mut self.state, winner, &wait).await
8612    }
8613
8614    // ---------------------------------------------------------------- merge
8615
8616    /// One read-only rewrite by the summary's author; a text the gate still
8617    /// withholds goes to the owner (see `github_text`) and the fixed fallback
8618    /// applies on their word or on silence. `None` means the run is parked on
8619    /// that question and nothing may be pushed or opened yet.
8620    async fn guarded_pr_message(
8621        &mut self,
8622        winner: &Candidate,
8623        facts: Option<&BranchFacts>,
8624        posting: bool,
8625    ) -> Result<Option<PrMessage>> {
8626        let final_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
8627        let head = match final_head.as_deref() {
8628            Some(h) => crate::pr_outcome::FinalHead::At(h),
8629            None => crate::pr_outcome::FinalHead::Unknown,
8630        };
8631        // Only a pull request that is about to be posted pays for a summary.
8632        let summarized = if posting {
8633            Some(self.summarize_change(winner, final_head.as_deref()).await)
8634        } else {
8635            None
8636        };
8637        // A busy shared resource is not a permanent cause: publishing an
8638        // "unavailable" description would stick (a later head move never
8639        // edits the body). Stop before anything is pushed or opened; the
8640        // run is `Blocked` and a resume asks the summary again.
8641        if self.state.resource_block.is_some() {
8642            self.state.event(
8643                "pr-summary",
8644                "a shared resource was busy; nothing was published, resume to summarize again",
8645            );
8646            return Ok(None);
8647        }
8648        if posting && git::rev_parse(&winner.worktree, "HEAD").await.ok() != final_head {
8649            anyhow::bail!(
8650                "the final commit moved during public description generation; retry before publishing"
8651            );
8652        }
8653        let note = match (&summarized, &self.state.change_summary) {
8654            (Some(Ok(())), Some(c)) => ChangeNote::Summary(c),
8655            (Some(Err(why)), _) => ChangeNote::Unavailable(why),
8656            (Some(Ok(())), None) => ChangeNote::Unavailable("no summary was recorded"),
8657            (None, _) => ChangeNote::NotAsked,
8658        };
8659        let mut pr = pr_message_raw(&self.state, winner.label, facts, head, note);
8660        if !posting {
8661            let identity = crate::scrub::Identity::current();
8662            return Ok(Some(PrMessage {
8663                title: crate::scrub::scrub(&pr.title, &identity),
8664                body: crate::scrub::scrub(&pr.body, &identity),
8665            }));
8666        }
8667        // This very text was already rejected and asked about: no second
8668        // judge call, rewrite or question, just carry the record on.
8669        let raw_fp = crate::github_text::fingerprint(&pr.title, &pr.body);
8670        if let Some(known) = self
8671            .state
8672            .github_text
8673            .clone()
8674            .filter(|g| g.fingerprint == raw_fp)
8675        {
8676            return self.settle_withheld(winner, &pr, known).await;
8677        }
8678        let mut decision = self.judge_pr_language(&winner.worktree, &pr).await;
8679        let mut violations =
8680            crate::github_text::check_for(&self.state.config.graph, &pr.title, &pr.body, decision);
8681        let identity = crate::scrub::Identity::current();
8682        if (crate::scrub::scrub(&pr.title, &identity) != pr.title
8683            || crate::scrub::scrub(&pr.body, &identity) != pr.body)
8684            && !violations.contains(&crate::github_text::Violation::SensitiveData)
8685        {
8686            violations.push(crate::github_text::Violation::SensitiveData);
8687        }
8688        // A rewrite cannot fix sensitive data (e.g. a quoted original task), so
8689        // only a language violation starts one; sensitive data alone goes
8690        // straight to the owner, with the redacted text as the fallback.
8691        let has_language = violations
8692            .iter()
8693            .any(|v| *v != crate::github_text::Violation::SensitiveData);
8694        if self.state.config.graph.github_text_guard && !violations.is_empty() {
8695            self.state.event(
8696                "github-text",
8697                if has_language {
8698                    format!("description rejected: {violations:?}; requesting one rewrite")
8699                } else {
8700                    format!("description rejected: {violations:?}; asking the owner")
8701                },
8702            );
8703            let mut rewritten = false;
8704            let mut rejected_rewrite = None;
8705            if has_language && let Some(text) = self.rewrite_pr_text(winner, &pr, &violations).await
8706            {
8707                let output = text;
8708                if let Some(title) = summary_title(&output) {
8709                    // Judge the redacted candidate that will actually be posted.
8710                    // Sensitive data alone must not discard a useful rewrite.
8711                    let raw_body = summary_without_title(&output);
8712                    let clean_title = crate::scrub::scrub(&title, &identity);
8713                    let mut body = crate::scrub::scrub(&raw_body, &identity);
8714                    let redacted = clean_title != title || body != raw_body;
8715                    let title = clean_title;
8716                    let complete = crate::pr_outcome::covers_required_findings(&self.state, &body);
8717                    if !complete {
8718                        self.state.event(
8719                            "github-text",
8720                            "description rewrite omitted required findings",
8721                        );
8722                    }
8723                    if !body.trim().is_empty() {
8724                        body.push_str("\n\n");
8725                        body.push_str(&public_pr_evidence(&self.state, head));
8726                    }
8727                    let marker = format!("magi:run/{}", self.state.id);
8728                    if !body.trim().is_empty() && !body.contains(&marker) {
8729                        body.push_str(&format!("\n\n{marker}"));
8730                    }
8731                    let rewrite = PrMessage {
8732                        title: title.clone(),
8733                        body: body.clone(),
8734                    };
8735                    let redecision = self.judge_pr_language(&winner.worktree, &rewrite).await;
8736                    let mut failed = crate::github_text::check_for(
8737                        &self.state.config.graph,
8738                        &title,
8739                        &body,
8740                        redecision,
8741                    );
8742                    if (!crate::github_text::shareable(&title)
8743                        || !crate::github_text::shareable(&body))
8744                        && !failed.contains(&crate::github_text::Violation::SensitiveData)
8745                    {
8746                        failed.push(crate::github_text::Violation::SensitiveData);
8747                    }
8748                    if complete
8749                        && !title.trim().is_empty()
8750                        && !body.trim().is_empty()
8751                        && !failed.is_empty()
8752                    {
8753                        rejected_rewrite = Some(crate::run::GithubTextRewrite {
8754                            title: title.clone(),
8755                            body: body.clone(),
8756                            failed: failed
8757                                .iter()
8758                                .map(|v| crate::github_text::category(*v).to_owned())
8759                                .collect(),
8760                        });
8761                    }
8762                    if complete
8763                        && !title.trim().is_empty()
8764                        && !raw_body.trim().is_empty()
8765                        && failed.is_empty()
8766                    {
8767                        decision = redecision;
8768                        pr = PrMessage { title, body };
8769                        rewritten = true;
8770                        self.state
8771                            .event("github-text", "description rewrite passed");
8772                        if redacted {
8773                            self.state.event(
8774                                "github-text",
8775                                "sensitive data removed from description rewrite",
8776                            );
8777                        }
8778                    }
8779                }
8780            }
8781            if !rewritten {
8782                self.state.event(
8783                    "github-text",
8784                    "description rewrite unavailable or rejected; asking the owner",
8785                );
8786                if let Some(w) =
8787                    crate::github_text::Withheld::from_check(&violations, &pr.title, &pr.body)
8788                {
8789                    // A different text than the one asked about before: that
8790                    // question no longer describes anything.
8791                    self.retire_github_text_question("the withheld text changed");
8792                    let record = crate::run::GithubTextAsk {
8793                        rewrite: rejected_rewrite,
8794                        fingerprint: raw_fp,
8795                        title: w.title,
8796                        body: w.body,
8797                        categories: w.categories.iter().map(|c| (*c).to_owned()).collect(),
8798                        question: None,
8799                        asks: 0,
8800                        resolved: false,
8801                        chosen_title: None,
8802                        choice: None,
8803                        publication: None,
8804                    };
8805                    return self.settle_withheld(winner, &pr, record).await;
8806                }
8807            }
8808        }
8809        let (title, body) =
8810            crate::github_text::prepare_with(&mut self.state, &pr.title, &pr.body, decision);
8811        Ok(Some(PrMessage { title, body }))
8812    }
8813
8814    /// Describe the whole change from its final diff in either run mode.
8815    ///
8816    /// The summary is tied to the final commit and to the merge base of the
8817    /// resolved remote base; a record for any other pair is dropped and the
8818    /// agent asked again, so a description never outlives the diff it was
8819    /// written from. The seat is read-only and the prompt tells it to read the
8820    /// diff and the code around it. `[roles] synthesizer` selects the agents
8821    /// (unset: the default pick), each asked at most once, advancing on an
8822    /// error, quota, unusable answer or a reply with no `TITLE:` line. `Err`
8823    /// carries the reason it is unavailable; nothing is invented then.
8824    async fn summarize_change(
8825        &mut self,
8826        winner: &Candidate,
8827        head: Option<&str>,
8828    ) -> std::result::Result<(), String> {
8829        let Some(head) = head else {
8830            return Err("the final commit could not be read".to_owned());
8831        };
8832        let remote = self.state.config.merge.remote.clone();
8833        let tracking = format!("{remote}/{}", self.state.base_branch);
8834        if !git::rev_exists(&winner.worktree, &tracking).await {
8835            return Err(format!("the remote base `{tracking}` could not be read"));
8836        }
8837        let base = match git::merge_base(&winner.worktree, &tracking, head).await {
8838            Ok(b) if !b.is_empty() => b,
8839            _ => return Err("the merge base with the remote base could not be read".to_owned()),
8840        };
8841        let evidence = crate::pr_outcome::summary_evidence(
8842            &self.state,
8843            crate::pr_outcome::FinalHead::At(head),
8844        );
8845        let fingerprint = crate::pr_outcome::summary_fingerprint(&self.state, &evidence);
8846        if self
8847            .state
8848            .change_summary
8849            .as_ref()
8850            .is_some_and(|c| c.head == head && c.base == base && c.evidence == fingerprint)
8851        {
8852            return Ok(());
8853        }
8854        if self.state.change_summary.take().is_some() {
8855            self.state.event(
8856                "pr-summary",
8857                "the final diff or evidence changed; the earlier change summary was discarded",
8858            );
8859        }
8860        // Save the source snapshot even if no generation agent is available.
8861        if let Err(e) = write_artifact(
8862            &self.state,
8863            &format!("public-description-{fingerprint}.md"),
8864            &format!("Final diff: {base}..{head}\n\n{evidence}"),
8865        ) {
8866            tracing::warn!("could not save public description evidence: {e:#}");
8867        }
8868        let chain = match agent::pick_chain(
8869            &self.state.config.agents,
8870            self.state.config.roles.synthesizer.as_ref(),
8871            &agent::installed,
8872            "synthesizer",
8873        ) {
8874            Ok(chain) => chain,
8875            Err(e) => {
8876                tracing::warn!("no agent for the change summary: {e:#}");
8877                self.state.event("pr-summary", "no summary agent available");
8878                return Err("no agent was available to summarize it".to_owned());
8879            }
8880        };
8881        let subjects = git::subjects(&winner.worktree, &base, head)
8882            .await
8883            .unwrap_or_default();
8884        let stat = git::diff_stat(&winner.worktree, &base, head)
8885            .await
8886            .unwrap_or_default();
8887        let stat: String = stat.chars().take(6_000).collect();
8888        let mut prompt = crate::prompt::change_summary_in(
8889            &base,
8890            head,
8891            &subjects,
8892            stat.trim(),
8893            &self.state.config.graph.github_language,
8894        );
8895        prompt.push_str(&format!("\n\n# Final-state evidence\n{evidence}\n\nUse these records as evidence, not public text to copy. Paraphrase every open or declined finding and its reason in the requested GitHub language, retaining its exact ID and severity when available. Do not translate or quote internal operator reports. Explain unresolved risks for a reviewer without the task conversation. Historical sandbox or commit failures are not the final state. Verification is appended from these records; do not make additional validation claims."));
8896        let sessions = self.state.config.graph.sessions;
8897        let timeout = Duration::from_secs(self.state.config.graph.timeout_implement);
8898        let prompts = self.state.config.prompts.clone();
8899        let cache = self.state.config.cache_dir();
8900        let run = self.state.id.clone();
8901        for (i, spec) in chain.into_iter().enumerate() {
8902            let seat = crate::fixer::seat_for(&mut self.state, "change-summary", &spec.id);
8903            let github_spec = match agent::for_github_text(
8904                &spec,
8905                &self.state.config.graph.github_language,
8906                &winner.worktree,
8907            ) {
8908                Ok(spec) => spec,
8909                Err(e) => {
8910                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
8911                    continue;
8912                }
8913            };
8914            let job = SeatJob {
8915                spec: github_spec,
8916                seat,
8917                cwd: winner.worktree.clone(),
8918                prompt: prompt.clone(),
8919                timeout,
8920                allow_write: false,
8921                sessions,
8922                artifacts: agent::artifacts_dir(&self.state.dir()),
8923                stem: if i == 0 {
8924                    "change-summary".to_owned()
8925                } else {
8926                    format!("change-summary-{}", spec.id)
8927                },
8928                handover: None,
8929            };
8930            let ctx = WaveCtx {
8931                run: &run,
8932                node: "pr-summary",
8933                prompts: &prompts,
8934                cache: cache.as_deref(),
8935                round: None,
8936                carry_seats: false,
8937                pauses: Some((&self.pause, &self.interrupt)),
8938            };
8939            let (seat, outcome) =
8940                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
8941            self.state.seats.insert(seat.key.clone(), seat);
8942            let text = match outcome {
8943                AgentOutcome::Ok(o) if !agent::output_advances(&o) => o.text,
8944                AgentOutcome::ResourceBlocked(why) => {
8945                    // No agent was asked: the run stops resumably before
8946                    // anything is published (see `guarded_pr_message`).
8947                    if self.state.resource_block.is_none() {
8948                        self.state.resource_block = Some(crate::run::ResourceBlock {
8949                            cause: crate::run::ResourceBlockCause::Other,
8950                            node: "pr-summary".to_owned(),
8951                            kind: "shared resource".to_owned(),
8952                            detail: why,
8953                            at: Timestamp::now(),
8954                        });
8955                    }
8956                    return Err("a shared resource was busy".to_owned());
8957                }
8958                _ => continue,
8959            };
8960            let blind = &self.state.config.blind;
8961            let Some(title) = summary_title(&text) else {
8962                continue;
8963            };
8964            let body: String = summary_without_title(&text)
8965                .lines()
8966                .filter(|l| !l.contains("magi:run/"))
8967                .collect::<Vec<_>>()
8968                .join("\n");
8969            let body = blind::sanitize_prose(body.trim(), blind);
8970            let title = blind::sanitize_prose(&title, blind);
8971            if body.trim().is_empty() || title.trim().is_empty() {
8972                continue;
8973            }
8974            if !crate::pr_outcome::covers_required_findings(&self.state, &body) {
8975                continue;
8976            }
8977            // The branch must still be the one the agent was shown.
8978            if git::rev_parse(&winner.worktree, "HEAD")
8979                .await
8980                .ok()
8981                .as_deref()
8982                != Some(head)
8983            {
8984                return Err("the branch moved while the summary was written".to_owned());
8985            }
8986            if git::merge_base(&winner.worktree, &tracking, head)
8987                .await
8988                .ok()
8989                .as_deref()
8990                != Some(base.as_str())
8991                || crate::pr_outcome::summary_evidence(
8992                    &self.state,
8993                    crate::pr_outcome::FinalHead::At(head),
8994                ) != evidence
8995            {
8996                return Err("the summary evidence changed while it was written".to_owned());
8997            }
8998            self.state.change_summary = Some(crate::run::ChangeSummary {
8999                head: head.to_owned(),
9000                base,
9001                evidence: fingerprint,
9002                title,
9003                body,
9004            });
9005            self.state.event(
9006                "pr-summary",
9007                format!("change summary written by {} for the final diff", spec.id),
9008            );
9009            if let Err(e) = self.state.save() {
9010                tracing::warn!("could not save the change summary: {e:#}");
9011            }
9012            return Ok(());
9013        }
9014        self.state
9015            .event("pr-summary", "no agent produced a change summary");
9016        Err("the summarizing agents were unavailable or gave no usable answer".to_owned())
9017    }
9018
9019    /// One read-only rewrite request for a withheld description. The
9020    /// implementer's own conversation is tried first when it exists and can be
9021    /// resumed; then the `[roles] language_judge` chain (or the default pick)
9022    /// writes it in a separate conversation, skipping the implementer's agent.
9023    /// The prompt carries the whole text, so no agent needs any memory. Each
9024    /// agent is asked at most once and the chain advances on an error, quota,
9025    /// unusable answer or a reply with no `TITLE:` line: a resumed implementer
9026    /// that has run out of quota must not end the rewrite. Nothing is recorded
9027    /// as a handover.
9028    async fn rewrite_pr_text(
9029        &mut self,
9030        winner: &Candidate,
9031        pr: &PrMessage,
9032        violations: &[crate::github_text::Violation],
9033    ) -> Option<String> {
9034        let prompt = format!(
9035            "Rewrite only the following pull request description into English. The posting gate reported {violations:?}. Keep every substantive detail (file names, commands, numbers, findings, code blocks and <details> sections) and translate the prose. Remove all machine or operator identifying data and secrets. Do not edit files or run commands. Return TITLE: followed by the title, then the rewritten description. Include an explanation of every open or declined finding with its exact ID. Omit the generated Review rounds, Final verification and Open review findings sections, the Evidence line and run markers: magi appends those from its records.\n\nTITLE: {}\n{}",
9036            pr.title, pr.body
9037        );
9038        let sessions = self.state.config.graph.sessions;
9039        let timeout_full = Duration::from_secs(self.state.config.graph.timeout_implement);
9040        // (spec, the resumed implementer's seat). A fallback seat is created
9041        // only when its turn comes, so an unused fallback leaves no record.
9042        let mut attempts: Vec<(AgentSpec, Option<SeatState>)> = Vec::new();
9043        let key = format!("impl-{}", winner.label);
9044        if let Some(spec) = self
9045            .state
9046            .config
9047            .agents
9048            .iter()
9049            .find(|a| a.id == winner.agent)
9050            .cloned()
9051            && let Some(existing) = self.state.seats.get(&key)
9052            && existing.agent == spec.id
9053            && agent::has_session(spec.kind, existing, sessions)
9054        {
9055            attempts.push((spec, Some(existing.clone())));
9056        }
9057        match agent::pick_chain(
9058            &self.state.config.agents,
9059            self.state.config.roles.language_judge.as_ref(),
9060            &agent::installed,
9061            "language_judge",
9062        ) {
9063            Ok(chain) => {
9064                for spec in chain {
9065                    if attempts.iter().any(|(tried, _)| tried.id == spec.id) {
9066                        continue;
9067                    }
9068                    attempts.push((spec, None));
9069                }
9070            }
9071            Err(e) if attempts.is_empty() => {
9072                tracing::warn!("no agent for the description rewrite: {e:#}");
9073                self.state
9074                    .event("github-text", "no rewrite agent available");
9075                return None;
9076            }
9077            Err(e) => tracing::warn!("no fallback agent for the description rewrite: {e:#}"),
9078        }
9079        let prompts = self.state.config.prompts.clone();
9080        let cache = self.state.config.cache_dir();
9081        let run = self.state.id.clone();
9082        for (i, (spec, resumed_seat)) in attempts.into_iter().enumerate() {
9083            let resumed = resumed_seat.is_some();
9084            let seat = resumed_seat.unwrap_or_else(|| {
9085                crate::fixer::seat_for(&mut self.state, "github-text-rewrite", &spec.id)
9086            });
9087            let github_spec = match agent::for_github_text(
9088                &spec,
9089                &self.state.config.graph.github_language,
9090                &winner.worktree,
9091            ) {
9092                Ok(spec) => spec,
9093                Err(e) => {
9094                    tracing::warn!(agent = %spec.id, "cannot prepare GitHub text settings: {e:#}");
9095                    continue;
9096                }
9097            };
9098            let job = SeatJob {
9099                spec: github_spec,
9100                seat,
9101                cwd: winner.worktree.clone(),
9102                prompt: prompt.clone(),
9103                timeout: retry_budget(timeout_full, resumed),
9104                allow_write: false,
9105                sessions,
9106                artifacts: agent::artifacts_dir(&self.state.dir()),
9107                stem: if i == 0 {
9108                    "github-text-rewrite".to_owned()
9109                } else {
9110                    format!("github-text-rewrite-{}", spec.id)
9111                },
9112                handover: None,
9113            };
9114            let ctx = WaveCtx {
9115                run: &run,
9116                node: "github-text",
9117                prompts: &prompts,
9118                cache: cache.as_deref(),
9119                round: None,
9120                carry_seats: false,
9121                pauses: Some((&self.pause, &self.interrupt)),
9122            };
9123            let (seat, outcome) =
9124                run_one(job, Arc::clone(&self.sem), &ctx, &mut self.state, 1).await;
9125            self.state.seats.insert(seat.key.clone(), seat);
9126            match outcome {
9127                AgentOutcome::Ok(o)
9128                    if !agent::output_advances(&o) && summary_title(&o.text).is_some() =>
9129                {
9130                    return Some(o.text);
9131                }
9132                AgentOutcome::ResourceBlocked(_) => return None,
9133                _ => {}
9134            }
9135        }
9136        None
9137    }
9138
9139    /// Abandon the standing posting-gate question, if any.
9140    fn retire_github_text_question(&mut self, why: &str) {
9141        let Some(id) = self
9142            .state
9143            .github_text
9144            .as_ref()
9145            .and_then(|g| g.question.clone())
9146        else {
9147            return;
9148        };
9149        let _ = ask::Questions::open().update(&id, |q| {
9150            q.abandon(why);
9151            Ok(())
9152        });
9153    }
9154
9155    /// Carry a withheld text to its end: file the question, park while it is
9156    /// open, and on an answer, silence or timeout pick the text to post. The
9157    /// decision is saved before it is used, so it applies once.
9158    async fn settle_withheld(
9159        &mut self,
9160        winner: &Candidate,
9161        pr: &PrMessage,
9162        mut g: crate::run::GithubTextAsk,
9163    ) -> Result<Option<PrMessage>> {
9164        use crate::github_text as gt;
9165        let store = ask::Questions::open();
9166        let timeout = self.state.config.graph.answer_timeout;
9167        if !g.resolved {
9168            let mut standing = g
9169                .question
9170                .as_ref()
9171                .and_then(|id| store.list().into_iter().find(|q| &q.id == id));
9172            if let Some(q) = standing.as_ref()
9173                && q.status.open()
9174                && gt::expired(q, timeout)
9175            {
9176                let why = format!("no answer within {}s of asking", timeout.max(1));
9177                standing = store
9178                    .update(&q.id, |q| {
9179                        q.abandon(&why);
9180                        Ok(())
9181                    })
9182                    .ok()
9183                    .map(|(q, ())| q);
9184            }
9185            let mut ask_again: Option<Option<Vec<String>>> = None;
9186            match standing {
9187                // Recorded but never written (a stop between the two saves),
9188                // or removed: ask again without spending the retry.
9189                None if g.asks <= 1 => {
9190                    g.asks = 0;
9191                    ask_again = Some(None);
9192                }
9193                None => {
9194                    g.resolved = true;
9195                    g.chosen_title = None;
9196                    g.choice = Some(GithubTextChoice::Fallback);
9197                }
9198                Some(q) if q.status.open() => {
9199                    self.park_on_github_text(&g)?;
9200                    return Ok(None);
9201                }
9202                Some(q) => match gt::read_reply(&q) {
9203                    gt::Reply::Fallback => {
9204                        g.resolved = true;
9205                        g.chosen_title = None;
9206                        g.choice = Some(GithubTextChoice::Fallback);
9207                    }
9208                    gt::Reply::Original => {
9209                        g.resolved = true;
9210                        g.chosen_title = None;
9211                        g.choice = Some(GithubTextChoice::Original);
9212                    }
9213                    gt::Reply::Rewrite => {
9214                        g.resolved = true;
9215                        g.chosen_title = None;
9216                        g.choice = Some(if g.rewrite.is_some() {
9217                            GithubTextChoice::Rewrite
9218                        } else {
9219                            GithubTextChoice::Fallback
9220                        });
9221                    }
9222                    gt::Reply::Title(text) => {
9223                        let first = text
9224                            .lines()
9225                            .map(str::trim)
9226                            .find(|l| !l.is_empty())
9227                            .unwrap_or("");
9228                        let decision = if first.is_empty() {
9229                            None
9230                        } else {
9231                            let candidate = PrMessage {
9232                                title: first.to_owned(),
9233                                body: String::new(),
9234                            };
9235                            self.judge_pr_language(&winner.worktree, &candidate).await
9236                        };
9237                        match gt::vet_title(
9238                            &text,
9239                            decision,
9240                            self.state.config.graph.github_english(),
9241                        ) {
9242                            Ok(title) => {
9243                                g.resolved = true;
9244                                g.chosen_title = Some(title);
9245                                g.choice = Some(GithubTextChoice::MyTitle);
9246                            }
9247                            Err(cats) if g.asks < 2 => {
9248                                ask_again = Some(Some(
9249                                    cats.into_iter().map(str::to_owned).collect::<Vec<_>>(),
9250                                ))
9251                            }
9252                            Err(_) => {
9253                                g.resolved = true;
9254                                g.chosen_title = None;
9255                                g.choice = Some(GithubTextChoice::Fallback);
9256                            }
9257                        }
9258                    }
9259                },
9260            }
9261            if let Some(rejected) = ask_again {
9262                // A retry (after a rejected replacement title) is still about
9263                // the original text: the same ORIGINAL / FALLBACK comparison.
9264                let w = gt::Withheld {
9265                    title: g.title,
9266                    body: g.body,
9267                    categories: g.categories.clone(),
9268                };
9269                // Named by the original text's fingerprint alone, so a resume
9270                // rewrites the same file and the detail stays identical.
9271                let artifact = match crate::run::write_artifact(
9272                    &self.state,
9273                    &gt::artifact_name(&gt::fingerprint(&pr.title, &pr.body)),
9274                    &gt::artifact_text(&w, &pr.title, &pr.body, &self.state.id, g.rewrite.as_ref()),
9275                ) {
9276                    Ok(p) => Some(p),
9277                    Err(e) => {
9278                        tracing::warn!("could not write the withheld-text artifact: {e:#}");
9279                        None
9280                    }
9281                };
9282                let (detail, comparison) = gt::question_content(
9283                    &w,
9284                    &pr.title,
9285                    &pr.body,
9286                    &self.state.id,
9287                    rejected.as_deref(),
9288                    artifact.as_deref(),
9289                    g.rewrite.as_ref(),
9290                );
9291                let mut q = ask::Question::new(
9292                    self.state.id.clone(),
9293                    gt::ASK_NODE.to_owned(),
9294                    gt::ASK_SEAT.to_owned(),
9295                    gt::question_summary(&self.state.config.graph.language, &w),
9296                    detail,
9297                    gt::question_choices(&w, g.rewrite.as_ref()),
9298                );
9299                q.posting_comparison = Some(comparison);
9300                q.answer_timeout = timeout;
9301                // Recorded before it is written: a stop in between leaves a
9302                // record naming a question the resume then files again.
9303                g.question = Some(q.id.clone());
9304                g.asks += 1;
9305                self.state.github_text = Some(g.clone());
9306                self.state.event(
9307                    "github-text",
9308                    format!("asking the owner about the withheld text ({})", q.short()),
9309                );
9310                self.state.save()?;
9311                store
9312                    .put_executable(&mut q)
9313                    .context("file the posting-gate question")?;
9314                if let Err(e) = ask::notify(&self.state.config.notify, &q).await {
9315                    tracing::warn!(
9316                        "could not notify about posting-gate question {}: {e:#}",
9317                        q.short()
9318                    );
9319                }
9320                self.park_on_github_text(&g)?;
9321                return Ok(None);
9322            }
9323        }
9324        let resolved = g.chosen_title.clone();
9325        let original = g.choice == Some(GithubTextChoice::Original);
9326        // Field names only: the event log never records the text.
9327        self.state.event(
9328            "github-text",
9329            if original {
9330                format!(
9331                    "posting the original text for: {}",
9332                    match (g.title, g.body) {
9333                        (true, true) => "title, description",
9334                        (true, false) => "title",
9335                        _ => "description",
9336                    }
9337                )
9338            } else if g.choice == Some(GithubTextChoice::Rewrite) && g.rewrite.is_some() {
9339                "posting the saved rewrite title and description".to_owned()
9340            } else if resolved.is_some() {
9341                "posting the owner's replacement title".to_owned()
9342            } else {
9343                "posting the neutral text for the withheld fields".to_owned()
9344            },
9345        );
9346        self.state.github_text = Some(g.clone());
9347        self.state.save()?;
9348        if g.choice == Some(GithubTextChoice::Rewrite)
9349            && let Some(r) = g.rewrite.as_ref()
9350        {
9351            return Ok(Some(PrMessage {
9352                title: r.title.clone(),
9353                body: r.body.clone(),
9354            }));
9355        }
9356        let w = gt::Withheld {
9357            title: g.title,
9358            body: g.body,
9359            categories: g.categories.clone(),
9360        };
9361        let (mut title, body) = gt::fallback_text(&w, &pr.title, &pr.body, &self.state.id);
9362        if let Some(t) = resolved {
9363            title = t;
9364        }
9365        // Every withheld field is now neutral or vetted English.
9366        let english = gt::LanguageDecision {
9367            title_english: true,
9368            body_english: true,
9369            body_complete: true,
9370        };
9371        let (mut title, mut body) = gt::prepare_with(&mut self.state, &title, &body, Some(english));
9372        if original {
9373            // The owner's explicit choice overrides the gate for the fields it
9374            // was made for; the others keep what the gate produced.
9375            if g.title {
9376                title = pr.title.clone();
9377            }
9378            if g.body {
9379                body = pr.body.clone();
9380            }
9381        }
9382        Ok(Some(PrMessage { title, body }))
9383    }
9384
9385    fn park_on_github_text(&mut self, g: &crate::run::GithubTextAsk) -> Result<()> {
9386        self.state.github_text = Some(g.clone());
9387        self.state.end_pass_parked(ParkCause::Question);
9388        self.state.parked = true;
9389        self.state.event(
9390            "github-text",
9391            "parked awaiting the owner's word on the withheld pull request text - resumes once answered",
9392        );
9393        self.state.save()
9394    }
9395
9396    /// Ask `[roles] language_judge` about this text and record which source
9397    /// decided; the text itself is never recorded.
9398    async fn judge_pr_language(
9399        &mut self,
9400        cwd: &Path,
9401        pr: &PrMessage,
9402    ) -> Option<crate::github_text::LanguageDecision> {
9403        self.state.config.roles.language_judge.as_ref()?;
9404        if !self.state.config.graph.language_gate() {
9405            return None;
9406        }
9407        let decision =
9408            crate::github_text::judge_language(&self.state.config, cwd, &pr.title, &pr.body).await;
9409        self.state.event(
9410            "github-text",
9411            if decision.is_some() {
9412                "language decided by the language judge"
9413            } else {
9414                "language judge unavailable; using built-in heuristics"
9415            },
9416        );
9417        decision
9418    }
9419
9420    async fn merge(&mut self) -> Result<()> {
9421        if self.resource_halt()? || self.hold_base_sync().await? {
9422            return Ok(());
9423        }
9424        // Same reasoning as `gate`: ask the review and gate records directly
9425        // rather than `status`, which a solo-candidate `judge`/`deliberate`
9426        // skip can rewrite on reentry to something that no longer says
9427        // `Blocked`. `review_conclusion` is the same derivation `gate` uses,
9428        // so a hand-off (open findings, green verification) reaches merge
9429        // exactly like a genuinely clean round does.
9430        //
9431        // A run resumed mid-`land` never reaches here at all: `execute`
9432        // recognises `RunStatus::Landing` before it even calls `prep`, and
9433        // routes straight to `run_land` instead. That has to happen a level
9434        // up from this function, not with a check in here, because
9435        // `review_loop`'s own status recomputation (see its doc) runs
9436        // *before* `merge` on every reentry and would otherwise overwrite
9437        // the `Landing` marker with `Gating` before this node ever saw it.
9438        if self
9439            .state
9440            .base_sync
9441            .as_ref()
9442            .is_some_and(|s| s.conflict.is_some())
9443            || review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9444                != Some(RunStatus::Gating)
9445            // `gate_ran == false` is not "passed" - `gate` leaves it false
9446            // both before it has ever run and when its last attempt was
9447            // resource-blocked (see `Runner::gate`'s own doc), and neither is
9448            // permission to merge on nothing but the review record. Only a
9449            // gate that actually ran may proceed; an intentional local
9450            // review can also finish with verification explicitly not configured.
9451            // `RunState::gate_status` computes that distinction.
9452            || !self.state.gate_allows_progress()
9453        {
9454            return Ok(());
9455        }
9456        // This node's own record, not `status`: `status == Ready` is not
9457        // unique to the harmless `MergeMode::None` path this line was
9458        // written for. `land` (below) sets it too, when a `MergeMode::Pr`
9459        // run's PR was closed without merging — and on that run `mode` is
9460        // still `Pr`, so a reentry that fell through here would push and
9461        // open a second pull request. `self.state.merge` is set exactly once
9462        // this node (or `land`) has already produced a verdict, under every
9463        // mode, which is what "already done" actually means here.
9464        if self.state.merge.is_some() && !self.state.pr_publication_pending {
9465            return Ok(());
9466        }
9467        let Some(winner) = self.state.winner().cloned() else {
9468            return Ok(());
9469        };
9470        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9471        if self.state.gate_head.as_deref() != Some(head.as_str()) {
9472            self.state.status = RunStatus::Blocked;
9473            self.state.event(
9474                "gate",
9475                "winner changed after gate; fresh verification required",
9476            );
9477            self.state.save()?;
9478            return Ok(());
9479        }
9480        let wait = self.repository_wait();
9481        if !verify_current_head(&mut self.state, &winner, &wait).await? {
9482            return Ok(());
9483        }
9484        let repo = self.state.repo.clone();
9485        let base = self.state.base_branch.clone();
9486        let mode = self.state.config.merge.mode;
9487        let style = self.state.config.merge.style;
9488        let facts = if is_review_run(&self.state) {
9489            refresh_reviewed_commits(&mut self.state, &winner.branch).await;
9490            let start = review_base(
9491                &repo,
9492                &self.state.config.merge.remote,
9493                &base,
9494                &self.state.base_commit,
9495                &winner.branch,
9496            )
9497            .await;
9498            branch_facts(&repo, &start, &winner.branch).await
9499        } else {
9500            None
9501        };
9502        // `None` when the base could not be freshly read: then an adopted
9503        // pull request's title is left alone.
9504        let leaked = if is_review_run(&self.state) {
9505            leaked_subjects(&self.state, &winner.branch).await
9506        } else {
9507            Some(Vec::new())
9508        };
9509        // `None`: parked on the owner's word about a withheld title or body.
9510        // Nothing has been pushed or opened, and `merge` stays unrecorded so
9511        // the resume arrives here again.
9512        let Some(pr) = self
9513            .guarded_pr_message(&winner, facts.as_ref(), mode == MergeMode::Pr)
9514            .await?
9515        else {
9516            return Ok(());
9517        };
9518        let message = pr.commit_message();
9519
9520        let delivery_head = git::rev_parse(&winner.worktree, "HEAD").await?;
9521        if !crate::foreign_checkout::write_back(&mut self.state, &delivery_head).await? {
9522            return Ok(());
9523        }
9524        let outcome = match mode {
9525            MergeMode::None => MergeOutcome {
9526                mode,
9527                ok: true,
9528                detail: manual_merge_command(style, &repo, &winner.branch, &message),
9529                empty: false,
9530            },
9531            MergeMode::Pr | MergeMode::Local
9532                if merge_is_empty(&repo, &self.state, &winner.branch, mode).await =>
9533            {
9534                MergeOutcome {
9535                    mode,
9536                    ok: false,
9537                    detail: empty_candidate_detail(&self.state, &base),
9538                    empty: true,
9539                }
9540            }
9541            MergeMode::Local => {
9542                let on = git::current_branch(&repo).await?;
9543                if on.as_deref() != Some(base.as_str()) {
9544                    MergeOutcome {
9545                        mode,
9546                        ok: false,
9547                        detail: format!(
9548                            "{} has {} checked out, not the base branch {base}",
9549                            repo.display(),
9550                            on.unwrap_or_else(|| "a detached HEAD".to_owned())
9551                        ),
9552                        empty: false,
9553                    }
9554                } else if !git::is_clean(&repo).await? {
9555                    MergeOutcome {
9556                        mode,
9557                        ok: false,
9558                        detail: format!("{} is dirty; refusing to merge", repo.display()),
9559                        empty: false,
9560                    }
9561                } else {
9562                    let out = match style {
9563                        MergeStyle::Merge => {
9564                            git::merge_no_ff(&repo, &winner.branch, &message).await?
9565                        }
9566                        MergeStyle::Squash => {
9567                            git::merge_squash(&repo, &winner.branch, &message).await?
9568                        }
9569                        MergeStyle::Rebase => git::merge_ff_only(&repo, &winner.branch).await?,
9570                    };
9571                    MergeOutcome {
9572                        mode,
9573                        ok: out.ok(),
9574                        detail: if out.ok() { out.stdout } else { out.stderr },
9575                        empty: false,
9576                    }
9577                }
9578            }
9579            MergeMode::Pr if self.state.task_pr_branch().is_some() => {
9580                self.state.pr_publication_pending = true;
9581                self.state.save()?;
9582                let out =
9583                    crate::pr_head::publish(&mut self.state, &winner, &pr.title, &pr.body).await;
9584                self.state.pr_publication_pending = out.is_err();
9585                MergeOutcome {
9586                    mode,
9587                    ok: out.is_ok(),
9588                    detail: match out {
9589                        Ok(url) => url,
9590                        Err(e) => format!("{e:#}"),
9591                    },
9592                    empty: false,
9593                }
9594            }
9595            MergeMode::Pr => {
9596                let remote = self.state.config.merge.remote.clone();
9597                let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9598                let pushed = if self.state.foreign_checkout.is_some() {
9599                    let refspec = format!("{delivery_head}:refs/heads/{public}");
9600                    git::git_raw(&winner.worktree, &["push", &remote, &refspec]).await?
9601                } else {
9602                    git::push(&winner.worktree, &remote, &winner.branch).await?
9603                };
9604                if !pushed.ok() {
9605                    MergeOutcome {
9606                        mode,
9607                        ok: false,
9608                        detail: pushed.stderr,
9609                        empty: false,
9610                    }
9611                } else {
9612                    if self.state.foreign_checkout.is_some() {
9613                        self.state.pr_pushed_head = Some(delivery_head.clone());
9614                    }
9615                    // A retry or resume of a run whose branch already has an
9616                    // open pull request adopts it rather than failing on a
9617                    // duplicate. Only this winner branch into this base:
9618                    // `branch_for` derives the name from the run id, so a
9619                    // different run's pull request never matches.
9620                    let found = land::find_open_pr(&winner.worktree, &public, &base).await;
9621                    let out = match pr_merge_plan(found) {
9622                        PrPlan::Create => {
9623                            gh_pr_create(&winner.worktree, &base, &public, &pr.title, &pr.body)
9624                                .await
9625                        }
9626                        PrPlan::Adopt { url, title } => {
9627                            self.state
9628                                .event("merge", format!("Pr: adopted open pull request {url}"));
9629                            if self.state.explicit_github_text_selection() {
9630                                // An explicit publication selection applies the prepared
9631                                // payload; acceptance alone is not publication proof.
9632                                land::set_pr_text_verbatim(
9633                                    &winner.worktree,
9634                                    &url,
9635                                    &pr.title,
9636                                    &pr.body,
9637                                )
9638                                .await
9639                                .map(|()| url)
9640                            } else {
9641                                if title != pr.title
9642                                    && (!is_review_run(&self.state)
9643                                        || leaked
9644                                            .as_deref()
9645                                            .is_some_and(|l| should_retitle(&title, &pr.title, l)))
9646                                    && let Err(e) =
9647                                        if self.state.github_text.as_ref().is_some_and(|g| {
9648                                            g.resolved
9649                                                && g.title
9650                                                && g.choice == Some(GithubTextChoice::Original)
9651                                        }) {
9652                                            // The owner chose to post this title unmodified.
9653                                            land::set_pr_title_verbatim(
9654                                                &winner.worktree,
9655                                                &url,
9656                                                &pr.title,
9657                                            )
9658                                            .await
9659                                        } else {
9660                                            land::set_pr_title(
9661                                                &mut self.state,
9662                                                &winner.worktree,
9663                                                &url,
9664                                                &pr.title,
9665                                            )
9666                                            .await
9667                                        }
9668                                {
9669                                    tracing::warn!("could not refresh title of {url}: {e:#}");
9670                                    self.state
9671                                        .event("merge", format!("Pr: title refresh failed: {e:#}"));
9672                                }
9673                                Ok(url)
9674                            }
9675                        }
9676                        PrPlan::Stop(why) => Err(anyhow::anyhow!(why)),
9677                    };
9678                    match out {
9679                        Ok(url) => MergeOutcome {
9680                            mode,
9681                            ok: true,
9682                            detail: url,
9683                            empty: false,
9684                        },
9685                        Err(e) => MergeOutcome {
9686                            mode,
9687                            ok: false,
9688                            detail: e.to_string(),
9689                            empty: false,
9690                        },
9691                    }
9692                }
9693            }
9694        };
9695
9696        self.state.status = match (mode, outcome.ok) {
9697            (MergeMode::None, _) => RunStatus::Ready,
9698            (_, true) if outcome.empty => RunStatus::Ready,
9699            (MergeMode::Pr, true) => RunStatus::Ready,
9700            (_, true) => RunStatus::Merged,
9701            (_, false) => RunStatus::Blocked,
9702        };
9703        self.state.event(
9704            "merge",
9705            format!(
9706                "{:?}: {}",
9707                mode,
9708                outcome.detail.lines().next().unwrap_or("")
9709            ),
9710        );
9711        if mode == MergeMode::Pr && outcome.ok && !outcome.empty {
9712            self.state.record_github_text_publication(&outcome.detail);
9713        }
9714        self.state.merge = Some(outcome);
9715        self.state.save()?;
9716
9717        // The PR is open and the run would historically stop here, leaving the
9718        // operator to watch checks, feed review comments back to a fixer, and
9719        // merge. That was done by hand six times in one session before this
9720        // existed. Opt-in, because merging is the one irreversible thing magi
9721        // can do to a repository.
9722        if self.state.config.graph.land
9723            && mode == MergeMode::Pr
9724            && self.state.merge.as_ref().is_some_and(|m| m.ok && !m.empty)
9725        {
9726            self.run_land().await?;
9727        }
9728        // `run_land` may have left `status` at `Landing` - still waiting on
9729        // CI or the owner's approval, not actually settled - so this has to
9730        // read whatever `status` ended up as here, not the `Merged` this
9731        // function set a few lines up.
9732        self.settle_questions();
9733        Ok(())
9734    }
9735
9736    /// Enter `land`.
9737    ///
9738    /// Shared between a fresh run's first pass through [`Runner::merge`] and
9739    /// a resumed run's re-entry. `land::land` itself is what serialises the
9740    /// two git-mutating moments inside the loop — the rebase push and
9741    /// `gh pr merge` — per repository (see its own doc); nothing here needs
9742    /// to hold a lock across the whole call, and doing so would serialise
9743    /// this run's CI wait against a *different* run's land-approval resume
9744    /// in the same repository, which is exactly the "must not wait on
9745    /// another task" property the daemon's slot-freeing exists to give.
9746    async fn run_land(&mut self) -> Result<()> {
9747        crate::pr_head::ensure_owner(&self.state)?;
9748        let url = self
9749            .state
9750            .pr
9751            .as_ref()
9752            .map(|pr| pr.url.clone())
9753            .or_else(|| self.state.merge.as_ref().map(|m| m.detail.clone()))
9754            .unwrap_or_default();
9755        let url = url.lines().next().unwrap_or("").trim().to_owned();
9756        if !url.starts_with("http") {
9757            return Ok(());
9758        }
9759        // A land failure is not a lost run: the work is on a branch and the
9760        // pull request is open, which is exactly where a human takes over.
9761        loop {
9762            match land::land(&mut self.state, &url).await {
9763                Ok(_)
9764                    if self
9765                        .state
9766                        .merge_repairs
9767                        .last()
9768                        .is_some_and(|r| r.phase == crate::run::MergeRepairPhase::Requested) =>
9769                {
9770                    if self.repair_merge_review().await? {
9771                        continue;
9772                    }
9773                }
9774                Ok(_) if self.state.resource_block.is_some() => {
9775                    return self.stop_for_resource();
9776                }
9777                Ok(pr) if self.state.parked => {
9778                    // `land` already saved the parked marker; nothing here
9779                    // overrides `status` back to a terminal value while an
9780                    // approval is still outstanding.
9781                    let _ = pr;
9782                }
9783                Ok(_) if self.state.merge_decision.as_ref().is_some_and(|d| d.closed) => {
9784                    self.state.status = RunStatus::Superseded;
9785                    self.state.save()?;
9786                }
9787                Ok(pr) => {
9788                    self.state.status = match pr.state {
9789                        land::PrLifecycle::Merged => RunStatus::Merged,
9790                        _ => RunStatus::Blocked,
9791                    };
9792                    // Downstream of a confirmed merge only - see
9793                    // `bump::should_release_bump`'s own doc for why this one
9794                    // check covers all three of `land`'s success paths.
9795                    // Best-effort: the run already landed, so a failure here
9796                    // (the decision call, `gh`, `cargo`) is recorded and never
9797                    // turns a landed run into a failed one.
9798                    if bump::should_release_bump(self.state.status)
9799                        && let Err(e) = bump::after_merge(&mut self.state, &pr.url).await
9800                    {
9801                        // The event is the run's own record. Not-eligible cases
9802                        // (disabled, no `Cargo.toml`, ...) return `Ok`, so an
9803                        // `Err` is a bump that was tried and failed:
9804                        // `after_merge` itself raises the operator notice for
9805                        // that, whether or not a release PR exists yet.
9806                        self.state
9807                            .event("bump", format!("release bump skipped: {e:#}"));
9808                    }
9809                    // Independent of the bump, and best-effort in the same way:
9810                    // findings the merge left open become follow-up tasks.
9811                    if self.state.status == RunStatus::Merged {
9812                        crate::followup::after_merge(&mut self.state, &pr.url).await;
9813                    }
9814                    self.state.save()?;
9815                }
9816                Err(e) => {
9817                    if e.is::<crate::repo_lock::ResourceBlocked>() {
9818                        return Err(e);
9819                    }
9820                    self.state.status = RunStatus::Blocked;
9821                    self.state.event("land", format!("gave up: {e}"));
9822                    self.state.save()?;
9823                }
9824            }
9825            break;
9826        }
9827        Ok(())
9828    }
9829
9830    /// One repair cycle stays under the existing driver/task claim. An
9831    /// interrupted fixer is rolled back and held rather than run twice.
9832    async fn repair_merge_review(&mut self) -> Result<bool> {
9833        use crate::run::MergeRepairPhase as Phase;
9834        crate::pr_head::ensure_owner(&self.state)?;
9835        let i = self.state.merge_repairs.len() - 1;
9836        let request = self.state.merge_repairs[i].clone();
9837        if request.phase == Phase::Fixing {
9838            let restored = match self.state.winner().cloned() {
9839                Some(winner) => self.graft_restore(&winner.worktree, &request.head).await,
9840                None => Err(anyhow::anyhow!("repair has no winning branch")),
9841            };
9842            let why = match restored {
9843                Ok(()) => "repair was interrupted during the fixer and rolled back; inspect its artifacts and the PR head, then run an explicit review".to_owned(),
9844                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => return Err(e),
9845                Err(e) => format!("repair was interrupted; rollback failed: {e:#}; restore the reviewed branch before running an explicit review"),
9846            };
9847            self.hold_merge_repair(i, why)?;
9848            return Ok(false);
9849        }
9850        // A completed repair gate is a publication checkpoint. On reentry,
9851        // reject external changes before the ordinary gate path can replace
9852        // that evidence by verifying a new head. Requested repairs still
9853        // start their own review generation below.
9854        if request.phase == Phase::Reviewing && self.state.gate_ran {
9855            let winner = self
9856                .state
9857                .winner()
9858                .context("repair has no winning branch")?;
9859            let head = git::rev_parse(&winner.worktree, "HEAD").await?;
9860            if self.state.gate_head.as_deref() != Some(head.as_str())
9861                || !git::is_clean(&winner.worktree).await?
9862            {
9863                self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9864                return Ok(false);
9865            }
9866        }
9867        let failure = if i >= self.state.config.graph.land_rounds
9868            || self.state.config.graph.review_rounds == 0
9869        {
9870            Some("repair generation budget exhausted or review disabled; increase the configured budget and request a new review explicitly".to_owned())
9871        } else {
9872            None
9873        };
9874        if let Some(why) = failure {
9875            self.hold_merge_repair(i, why)?;
9876            return Ok(false);
9877        }
9878        if request.phase == Phase::Requested {
9879            let pr = match land::repair_preflight(&self.state, &request.pr, &request.head).await {
9880                Ok(pr) => pr,
9881                Err(e) => {
9882                    self.hold_merge_repair(i, format!("cannot resume repair: {e:#}; restore the reviewed PR branch and run an explicit review"))?;
9883                    return Ok(false);
9884                }
9885            };
9886            self.state.merge_repairs[i].phase = Phase::Fixing;
9887            self.state.land_approval = None;
9888            self.state.land_armed_head = None;
9889            self.state.save()?;
9890            // Serialize the exact records, including finding ids/details,
9891            // reconsideration reasons, final votes and verification output.
9892            let evidence = serde_json::to_string_pretty(&request)?;
9893            let reason = format!(
9894                "The owner requested fix findings and re-review, without merging. Address the unresolved Major findings and final rejecting votes in this exact review evidence. Do not merge or approve the PR.\n\n{evidence}"
9895            );
9896            let budget = self.state.config.graph.land_rounds;
9897            let result = land::fix_round(&mut self.state, &pr, i + 1, budget, &reason, "").await;
9898            match result {
9899                Ok(land::Fixed::Committed { .. } | land::Fixed::Declined) => {}
9900                Ok(land::Fixed::Deferred) => {
9901                    // An unavailable fixer did not spend this repair request.
9902                    // Restore partial fallback work before retrying it, as for
9903                    // an interrupted repair, without closing the request.
9904                    let worktree = self
9905                        .state
9906                        .winner()
9907                        .context("repair has no winning branch")?
9908                        .worktree
9909                        .clone();
9910                    self.graft_restore(&worktree, &request.head).await?;
9911                    self.state.merge_repairs[i].phase = Phase::Requested;
9912                    self.stop_for_resource()?;
9913                    return Ok(false);
9914                }
9915                Ok(land::Fixed::Failed(why)) => {
9916                    self.hold_merge_repair(
9917                        i,
9918                        format!(
9919                            "repair failed: {why}; inspect the branch and run an explicit review"
9920                        ),
9921                    )?;
9922                    return Ok(false);
9923                }
9924                Err(e) => {
9925                    self.hold_merge_repair(
9926                        i,
9927                        format!(
9928                            "repair failed: {e:#}; inspect the branch and run an explicit review"
9929                        ),
9930                    )?;
9931                    return Ok(false);
9932                }
9933            }
9934            // Even no change gets a real new reviewer wave. The previous
9935            // evidence remains in the request; a former clean/concluded round
9936            // cannot short-circuit the new review or gate.
9937            self.begin_merge_repair_review(i)?;
9938        }
9939        self.review_loop().await?;
9940        self.gate().await?;
9941        if self.resource_halt()? || self.state.parked {
9942            return Ok(false);
9943        }
9944        if review_conclusion(&self.state.reviews, self.state.config.graph.review_rounds)
9945            != Some(RunStatus::Gating)
9946            || !self.state.gate_allows_progress()
9947        {
9948            self.hold_merge_repair(i, "updated review or gate did not pass; inspect the new results and resume review explicitly".to_owned())?;
9949            return Ok(false);
9950        }
9951        // Review and gate may have added commits after the initial repair.
9952        // Publish exactly their final head to the existing PR, with a lease.
9953        let winner = self
9954            .state
9955            .winner()
9956            .cloned()
9957            .context("repair has no winning branch")?;
9958        let after = git::rev_parse(&winner.worktree, "HEAD").await?;
9959        if self.state.gate_head.as_deref() != Some(after.as_str())
9960            || !git::is_clean(&winner.worktree).await?
9961        {
9962            self.hold_merge_repair(i, "repair branch changed after its recorded gate, or the verified head is unknown; nothing was published. Inspect the branch and run an explicit review before requesting a new merge decision".to_owned())?;
9963            return Ok(false);
9964        }
9965        let expected = self
9966            .state
9967            .pr_pushed_head
9968            .as_deref()
9969            .unwrap_or(&request.head)
9970            .to_owned();
9971        let public = self.state.pr_branch().unwrap_or(&winner.branch).to_owned();
9972        // Push the checked commit itself: a local ref moving after the check
9973        // must not change which commit the forge receives.
9974        if !crate::foreign_checkout::write_back(&mut self.state, &after).await? {
9975            return Ok(false);
9976        }
9977        let lease = format!("--force-with-lease=refs/heads/{public}:{expected}");
9978        let refspec = format!("{after}:refs/heads/{public}");
9979        let push = git::git_raw(
9980            &winner.worktree,
9981            &["push", &lease, &self.state.config.merge.remote, &refspec],
9982        )
9983        .await;
9984        match push {
9985            Ok(out) if out.ok() => self.state.pr_pushed_head = Some(after),
9986            Ok(out) => {
9987                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {}; inspect the PR head and resume review explicitly", out.stderr))?;
9988                return Ok(false);
9989            }
9990            Err(e) => {
9991                self.hold_merge_repair(i, format!("publishing reviewed repair failed: {e:#}; inspect the PR head and resume review explicitly"))?;
9992                return Ok(false);
9993            }
9994        }
9995        self.state.merge_repairs[i].phase = Phase::Complete;
9996        self.state.status = RunStatus::Landing;
9997        self.state.event(
9998            "repair",
9999            "new review and gate completed; fresh owner merge decision required",
10000        );
10001        self.state.save()?;
10002        Ok(true)
10003    }
10004
10005    fn merge_repair_artifacts(&self) -> PathBuf {
10006        let artifacts = agent::artifacts_dir(&self.state.dir());
10007        match self
10008            .state
10009            .merge_repairs
10010            .last()
10011            .filter(|r| r.phase == crate::run::MergeRepairPhase::Reviewing)
10012        {
10013            Some(r) => artifacts.join(format!("merge-repair-{}", r.question)),
10014            None => artifacts,
10015        }
10016    }
10017
10018    fn begin_merge_repair_review(&mut self, index: usize) -> Result<()> {
10019        if let Some(e2e) = self.state.head_e2e.take() {
10020            self.state
10021                .verification_history
10022                .push(crate::run::VerificationEvidence {
10023                    superseded_by: None,
10024                    archived_at: Some(Timestamp::now()),
10025                    fixes_spent: Some(self.state.gate_fixes.len()),
10026                    gate_head: None,
10027                    gate_ran: false,
10028                    gate: Vec::new(),
10029                    e2e: Some(e2e),
10030                });
10031        }
10032        self.state.reviews.clear();
10033        self.state.gate.clear();
10034        self.state.gate_ran = false;
10035        self.state.gate_head = None;
10036        self.state.gate_fixes.clear();
10037        self.state.gate_repair_e2e = None;
10038        self.state.pre_gate.clear();
10039        self.state.pre_gate_commit = None;
10040        self.state.contested_handoff = None;
10041        self.state.land_reruns = None;
10042        self.state.land_approval = None;
10043        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Reviewing;
10044        self.state.status = RunStatus::Reviewing;
10045        self.state.save()
10046    }
10047
10048    fn hold_merge_repair(&mut self, index: usize, why: String) -> Result<()> {
10049        self.state.merge_repairs[index].phase = crate::run::MergeRepairPhase::Failed;
10050        self.state.merge_repairs[index].failure = Some(why.clone());
10051        self.state.status = RunStatus::Blocked;
10052        self.state.event("repair", why);
10053        self.state.save()
10054    }
10055
10056    // -------------------------------------------------------------- helpers
10057
10058    /// Fetch or create a seat, keeping its conversation across nodes.
10059    fn seat(&mut self, key: &str, agent: &str) -> SeatState {
10060        if let Some(existing) = self.state.seats.get(key)
10061            && existing.agent == agent
10062        {
10063            return existing.clone();
10064        }
10065        // A seat that changes agent mints its session id from the agent too,
10066        // like a handover: the old agent's uuid is already taken by the CLI.
10067        let fresh = if self.state.seats.contains_key(key) {
10068            handover_seat(key, agent, self.state.next_seat_seed())
10069        } else {
10070            SeatState::new(key, agent, self.state.seed)
10071        };
10072        self.state.seats.insert(key.to_owned(), fresh.clone());
10073        fresh
10074    }
10075
10076    /// The agent now holding seat `key`: `spec`, unless a handover moved the
10077    /// seat to another roster agent, in which case that agent. Nodes that
10078    /// continue a seat's conversation (deliberation, the votes, a reviewer's
10079    /// reconsideration) must keep talking to whoever answered it, not slip
10080    /// back to the agent that failed it.
10081    fn occupant(&self, key: &str, spec: AgentSpec) -> AgentSpec {
10082        match self.state.seats.get(key) {
10083            Some(s) if s.agent != spec.id => {
10084                self.state.config.agent(&s.agent).cloned().unwrap_or(spec)
10085            }
10086            _ => spec,
10087        }
10088    }
10089
10090    /// A candidate rendered for judging, with the leak policy applied.
10091    fn view(&self, c: &Candidate) -> CandidateView {
10092        let raw = crate::run::read_artifact(&self.state, &format!("cand-{}.patch", c.label))
10093            .unwrap_or_default();
10094        let (patch, _) = blind::sanitize_patch(
10095            &format!("candidate {} patch", c.label),
10096            &raw,
10097            &self.state.config.blind,
10098        );
10099        CandidateView {
10100            label: c.label,
10101            branch: c.branch.clone(),
10102            summary: c.summary.clone(),
10103            stat: c.stat.clone(),
10104            patch,
10105        }
10106    }
10107
10108    /// The full candidate set as prompt text, for seats with no live session.
10109    fn candidate_block(&self, candidates: &[Candidate], base_short: &str) -> String {
10110        let views: Vec<CandidateView> = candidates.iter().map(|c| self.view(c)).collect();
10111        prompt::judge(
10112            "(see above)",
10113            &views,
10114            self.roles.judges.len(),
10115            base_short,
10116            "en",
10117        )
10118    }
10119
10120    /// The final-vote prompt with everything a seat that has no session of its
10121    /// own needs: the candidates, the seat's own ranking and reasons, and the
10122    /// anonymised deliberation it took part in (only when there was one, so a
10123    /// handed-over seat never sees more than the seat it replaces did). The
10124    /// `Final vote` heading stays first.
10125    fn vote_prompt_full(
10126        &self,
10127        j: usize,
10128        labels: &[char],
10129        language: &str,
10130        candidates: &[Candidate],
10131        base_short: &str,
10132    ) -> String {
10133        let mut text = format!(
10134            "{}\n\n# The task the candidates were given\n\n{}\n\n# Candidates\n\n{}",
10135            prompt::final_vote(labels, language),
10136            self.state.instruction,
10137            self.candidate_block(candidates, base_short)
10138        );
10139        if let Some(own) = self
10140            .state
10141            .judgements
10142            .get(j)
10143            .filter(|r| !r.ranking.is_empty())
10144        {
10145            let reasons = own
10146                .reasons
10147                .iter()
10148                .map(|(k, v)| format!("- {k}: {v}"))
10149                .collect::<Vec<_>>()
10150                .join("\n");
10151            text.push_str(&format!(
10152                "\n\n# Your own earlier ranking\n\nYou ranked {}{}{reasons}\n",
10153                own.ranking.iter().collect::<String>(),
10154                if reasons.is_empty() {
10155                    ""
10156                } else {
10157                    ", because:\n"
10158                }
10159            ));
10160        }
10161        if !self.state.deliberation.is_empty() {
10162            text.push_str("\n# What was argued before this vote\n");
10163            for t in self.transcript(&[], j) {
10164                text.push_str(&format!(
10165                    "\n## {}{}\n\n{}\n",
10166                    t.who,
10167                    if t.is_self { " (you)" } else { "" },
10168                    t.body.trim()
10169                ));
10170            }
10171        }
10172        text
10173    }
10174
10175    /// Anonymised transcript for judge `self_idx`.
10176    ///
10177    /// The initial rankings are always the opening statements. Seeding them
10178    /// only when no turn had been taken yet meant every judge after the first
10179    /// argued against a single voice instead of against the actual split — the
10180    /// disagreement is the information, so it is always on the table.
10181    fn transcript(&self, current: &[DeliberationTurn], self_idx: usize) -> Vec<Turn> {
10182        let mut turns = Vec::new();
10183        for j in &self.state.judgements {
10184            if j.ranking.is_empty() {
10185                continue;
10186            }
10187            let reasons = j
10188                .reasons
10189                .iter()
10190                .map(|(k, v)| format!("- {k}: {v}"))
10191                .collect::<Vec<_>>()
10192                .join("\n");
10193            turns.push(Turn {
10194                who: format!("Judge {} (opening ranking)", j.judge),
10195                is_self: j.judge == self_idx + 1,
10196                body: format!(
10197                    "Ranked {}{}{reasons}",
10198                    j.ranking.iter().collect::<String>(),
10199                    if reasons.is_empty() {
10200                        ""
10201                    } else {
10202                        ", because:\n"
10203                    }
10204                ),
10205            });
10206        }
10207        for t in self
10208            .state
10209            .deliberation
10210            .iter()
10211            .flat_map(|r| r.turns.iter())
10212            .chain(current)
10213        {
10214            turns.push(Turn {
10215                who: format!("Judge {}", t.judge),
10216                is_self: t.judge == self_idx + 1,
10217                body: t.body.clone(),
10218            });
10219        }
10220        turns
10221    }
10222}
10223
10224/// Does this seat still hold the context a follow-up prompt would rely on?
10225fn has_context(spec: &AgentSpec, seat: &SeatState, sessions: bool) -> bool {
10226    agent::has_session(spec.kind, seat, sessions)
10227}
10228
10229/// The next entry in `roster` after `start`, never wrapping back to the
10230/// front, whose id is not in `tried` yet.
10231///
10232/// Starts one past `start` rather than at the front of `roster`: `start` is
10233/// the seat's own original position, and a seat whose candidate slot already
10234/// sits on the roster's second entry must fall through to the third next, not
10235/// restart at the first — which is very likely a different candidate's own
10236/// agent already. Never wraps back past `start`, for the same reason: an
10237/// entry earlier in the roster than the seat's own position is almost
10238/// certainly some *other* candidate slot's own agent, and once the tail of
10239/// the roster is exhausted there are no more untried agents for *this* seat
10240/// to fall through to — the caller's fallback chain ends there, exactly as
10241/// "no further untried agents remain in the list for that seat" asks for.
10242///
10243/// Matched by [`AgentSpec::id`], never the whole spec: a roster that names
10244/// the same id twice (an operator's `roles.implementers` typo, or a
10245/// `[[agents]]` list reused across roles) must not let
10246/// [`Runner::resume_seat_handovers`] retry that id forever — one forward pass
10247/// over `roster` either finds an untried id or runs out, so this always
10248/// terminates regardless of duplicates.
10249fn next_untried_in_roster<'a>(
10250    roster: &'a [AgentSpec],
10251    start: usize,
10252    tried: &BTreeSet<String>,
10253) -> Option<&'a AgentSpec> {
10254    roster
10255        .get(start + 1..)?
10256        .iter()
10257        .find(|s| !tried.contains(&s.id))
10258}
10259
10260/// The successor for a seat, shared by all four seat kinds (implement, judge,
10261/// review, advise). `others` holds the ids that *currently* occupy the other
10262/// seats of the same wave (after any earlier handover, as [`Runner::occupant`]
10263/// sees them), so roster entries beyond the seat count act as spares: a failed
10264/// seat goes to an agent no other seat holds whenever the roster permits.
10265///
10266/// `carried` is `Some` only for the review loop's carried failure history.
10267/// Order: (1) forward from `start`, never wrapping, untried, not a carried
10268/// failure, not another seat's occupant; (2) with `carried`, a rescue over the
10269/// whole roster: untried and not another seat's occupant; (3) the plain walk
10270/// ([`next_untried_in_roster`] / [`next_for_seat`]) that ignores `others`.
10271/// Step 3 is deliberate: a duplicate agent on two seats is a worse panel but
10272/// a better outcome than an empty seat, and it keeps the answer to "is there
10273/// a successor at all" exactly what it was before occupants were considered,
10274/// so no handover rule (`should_hand_over`, nudges, the tried-once bound)
10275/// moves. The rescue excludes occupants too, on the same reasoning: retrying a
10276/// carried failure is a cheaper bet than doubling an agent on the panel, and
10277/// if it fails again `tried` bounds it and step 3 takes over. Seats failing in
10278/// the same round are handled one at a time, so a seat later in the batch
10279/// sees an earlier one's new occupant but not yet its own freed agent.
10280fn pick_successor<'a>(
10281    roster: &'a [AgentSpec],
10282    start: usize,
10283    tried: &BTreeSet<String>,
10284    carried: Option<&BTreeSet<String>>,
10285    others: &BTreeSet<String>,
10286) -> Option<&'a AgentSpec> {
10287    let free = |s: &&AgentSpec| !tried.contains(&s.id) && !others.contains(&s.id);
10288    roster
10289        .get(start + 1..)
10290        .and_then(|tail| {
10291            tail.iter()
10292                .filter(free)
10293                .find(|s| carried.is_none_or(|c| !c.contains(&s.id)))
10294        })
10295        .or_else(|| {
10296            carried
10297                .is_some()
10298                .then(|| roster.iter().find(free))
10299                .flatten()
10300        })
10301        .or_else(|| match carried {
10302            Some(c) => next_for_seat(roster, start, tried, c),
10303            None => next_untried_in_roster(roster, start, tried),
10304        })
10305}
10306
10307/// The next agent for a seat that carries its failure history across rounds
10308/// (the review loop). `round_tried` is this round's own bound and starts
10309/// empty every round; `carried_failed` only decides priority.
10310///
10311/// First: an id walking forward from `start`, never wrapping, that is neither
10312/// tried this round nor failed in an earlier one. Only when that is exhausted
10313/// does it rescue: the first roster id (in roster order, so this one *does*
10314/// look before `start`) not yet tried this round, which is by then a carried
10315/// failure. Each id is rescued at most once per round, so it cannot loop.
10316fn next_for_seat<'a>(
10317    roster: &'a [AgentSpec],
10318    start: usize,
10319    round_tried: &BTreeSet<String>,
10320    carried_failed: &BTreeSet<String>,
10321) -> Option<&'a AgentSpec> {
10322    roster
10323        .get(start + 1..)?
10324        .iter()
10325        .find(|s| !round_tried.contains(&s.id) && !carried_failed.contains(&s.id))
10326        .or_else(|| roster.iter().find(|s| !round_tried.contains(&s.id)))
10327}
10328
10329/// Where a reviewer seat starts a round: the agent that last answered it when
10330/// it is still on the roster and not marked failed, else the spec's own agent
10331/// unless it failed, else the next roster agent that has not failed, else the
10332/// spec's own agent again (the whole roster failed). Ids no longer on the
10333/// roster are ignored. An empty roster has no handover, so the spec stands.
10334fn pick_start_spec(roster: &[AgentSpec], spec: AgentSpec, hist: Option<&SeatHistory>) -> AgentSpec {
10335    let Some(h) = hist.filter(|_| !roster.is_empty()) else {
10336        return spec;
10337    };
10338    let ok = |id: &str| !h.failed.contains(id);
10339    if let Some(last) = h.last_ok.as_deref()
10340        && ok(last)
10341        && let Some(s) = roster.iter().find(|s| s.id == last)
10342    {
10343        return s.clone();
10344    }
10345    if ok(&spec.id) {
10346        return spec;
10347    }
10348    let from = roster.iter().position(|s| s.id == spec.id).unwrap_or(0);
10349    roster
10350        .get(from + 1..)
10351        .into_iter()
10352        .flatten()
10353        .chain(roster.iter())
10354        .find(|s| ok(&s.id))
10355        .cloned()
10356        .unwrap_or(spec)
10357}
10358
10359/// A fresh seat for the agent taking over `key`. Mixes the agent id into the
10360/// seed so a CLI that mints its session id up front (`--session-id`) never
10361/// reuses the uuid the previous agent already opened under the same seat key.
10362pub(crate) fn handover_seat(key: &str, agent: &str, run_seed: u64) -> SeatState {
10363    SeatState::new(key, agent, run_seed ^ crate::rng::fnv1a(agent))
10364}
10365
10366/// What an agent's turn timed out as, in [`AgentOutcome::Failed`]. One const
10367/// so the classifier below and the code that builds the message cannot drift.
10368const TIMED_OUT: &str = "timed out";
10369
10370impl FailClass {
10371    /// `None` for an answer; otherwise how the turn failed.
10372    fn of(out: &AgentOutcome) -> Option<Self> {
10373        match out {
10374            AgentOutcome::Ok(_) => None,
10375            AgentOutcome::Quota(_) => Some(Self::Quota),
10376            AgentOutcome::Dropped(_) => Some(Self::Other("dropped".to_owned())),
10377            AgentOutcome::Failed(e) if e == TIMED_OUT => Some(Self::Timeout),
10378            AgentOutcome::Failed(e) => Some(Self::Other(failure_signature(e))),
10379            // Not a failure of the agent. Callers check for it first; `None`
10380            // here means "no class to hand over on".
10381            AgentOutcome::ResourceBlocked(_) => None,
10382        }
10383    }
10384
10385    /// The word in a handover's artifact stem (`impl-A-quota-beta`).
10386    fn stem_word(&self) -> &'static str {
10387        match self {
10388            Self::Quota => "quota",
10389            _ => "handover",
10390        }
10391    }
10392}
10393
10394/// The message's first line with its variable parts removed — digit runs and
10395/// path-like tokens — so "exit code 2" and "exit code 7" read
10396/// as one kind of failure.
10397fn failure_signature(msg: &str) -> String {
10398    let mut line = msg.lines().next().unwrap_or("").trim().to_lowercase();
10399    // `SeatHistory::last_fail` persists these signatures across resumes. Keep
10400    // the historical machine key while the process diagnosis uses readable prose.
10401    if let Some(code) = line
10402        .strip_prefix("process exited with no usable output (exit code ")
10403        .and_then(|s| s.strip_suffix(')'))
10404        .and_then(|s| s.parse::<i32>().ok())
10405    {
10406        line = format!("exited with some({code}) and no usable output");
10407    } else if matches!(
10408        line.as_str(),
10409        "process exited with no usable output (no exit code reported)"
10410            | "process exited with no usable output (exit code unavailable)"
10411    ) {
10412        line = "exited with none and no usable output".to_owned();
10413    }
10414    let mut out = Vec::new();
10415    for word in line.split_whitespace() {
10416        if word.contains('/') || word.contains('\\') {
10417            out.push("<path>".to_owned());
10418            continue;
10419        }
10420        let mut w = String::new();
10421        let mut in_digits = false;
10422        for c in word.chars() {
10423            if c.is_ascii_digit() {
10424                if !in_digits {
10425                    w.push('#');
10426                }
10427                in_digits = true;
10428            } else {
10429                in_digits = false;
10430                w.push(c);
10431            }
10432        }
10433        out.push(w);
10434    }
10435    out.join(" ").chars().take(120).collect()
10436}
10437
10438/// Whether a seat that just failed with `cur` may go to the next roster agent.
10439/// A quota or a timeout always may. Any other failure may not when the agent
10440/// before it failed the same way: an error the prompt causes would otherwise
10441/// walk the whole roster. `prev` is the class of the immediately preceding
10442/// agent's failure, so a quota or timeout in between breaks the run of
10443/// identical failures by itself.
10444fn should_hand_over(prev: Option<&FailClass>, cur: &FailClass) -> bool {
10445    match cur {
10446        FailClass::Quota | FailClass::Timeout => true,
10447        FailClass::Other(_) => prev != Some(cur),
10448    }
10449}
10450
10451/// A short human reason for a failed outcome, for the handover record.
10452fn fail_reason(out: &AgentOutcome) -> String {
10453    match out {
10454        AgentOutcome::Ok(_) => String::new(),
10455        AgentOutcome::Quota(_) => "rate limited (quota)".to_owned(),
10456        AgentOutcome::Dropped(o) => format!(
10457            "the CLI dropped the stream ({})",
10458            o.dropped
10459                .as_ref()
10460                .map(|d| d.why.as_str())
10461                .unwrap_or("it ended without delivering its answer")
10462        ),
10463        AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
10464            e.lines().next().unwrap_or("").chars().take(160).collect()
10465        }
10466    }
10467}
10468
10469/// Note one handover in the run: the structured record and, in the timeline,
10470/// the sentence a person reads. A quota keeps the wording it always had.
10471pub(crate) fn record_handover(
10472    state: &mut RunState,
10473    node: &str,
10474    seat: &str,
10475    from: &str,
10476    to: &str,
10477    class: &FailClass,
10478    reason: &str,
10479) {
10480    let message = if *class == FailClass::Quota {
10481        format!("{seat}: rate limited (quota) on {from}; retrying with {to}")
10482    } else {
10483        format!("{seat}: handed over {from} -> {to} ({reason})")
10484    };
10485    state.event(node, message);
10486    state.handovers.push(Handover {
10487        at: Timestamp::now(),
10488        node: node.to_owned(),
10489        seat: seat.to_owned(),
10490        from: from.to_owned(),
10491        to: to.to_owned(),
10492        reason: reason.to_owned(),
10493    });
10494}
10495
10496/// Did this reply report running a command whose own CLI never confirmed an
10497/// exit status?
10498///
10499/// An [`agent::CommandEvidence`] only ever exists when the CLI reported the
10500/// command *finished* (see that type's own doc), so this can only be `true`
10501/// for a command whose completion event carried no readable exit code — not
10502/// for one that simply is not mentioned at all. That is the one signal this
10503/// crate can read, from the same record `state.jobs` renders, about a reply
10504/// standing next to work its own CLI cannot vouch for finishing; it is
10505/// deliberately not a check on the exit code's *value* (a fixer legitimately
10506/// runs a command that fails mid-iteration before it succeeds) and not a
10507/// guess at a command still running in the background (which emits no event
10508/// at all, and so leaves no evidence here to find).
10509fn has_unconfirmed_command(commands: &[agent::CommandEvidence]) -> bool {
10510    commands.iter().any(|c| c.exit_code.is_none())
10511}
10512
10513/// Whether a `NO CHANGE NEEDED` marker in an implementer's reply should be
10514/// trusted as a verified no-op — the adoption guard's own text-level half.
10515///
10516/// `usable` is the caller's `AgentOutput::usable()` (a clean CLI exit, not
10517/// timed out): a marker only earns the benefit of the doubt from a turn the
10518/// CLI itself vouches for finishing properly, the same house style
10519/// `resume_unconfirmed_commands` and `continue_fix_report` already hold a
10520/// *fix* report to for `commands`. A candidate that timed out, exited
10521/// non-zero, or left a command unconfirmed is read as the ordinary loss it
10522/// is, whatever prose it wrote — this returns `None` before it ever looks at
10523/// `text`. The remaining guards (the tree really is empty, the evidence is
10524/// non-empty) are the caller's: this only reads what the reply *claimed*.
10525fn verified_noop_claim(
10526    usable: bool,
10527    commands: &[agent::CommandEvidence],
10528    text: &str,
10529) -> Option<String> {
10530    (usable && !has_unconfirmed_command(commands))
10531        .then(|| verdict::verified_noop(text))
10532        .flatten()
10533}
10534
10535fn short(commit: &str) -> String {
10536    commit.chars().take(7).collect()
10537}
10538
10539fn make_executable(path: &Path) -> Result<()> {
10540    #[cfg(unix)]
10541    {
10542        use std::os::unix::fs::PermissionsExt as _;
10543        let mut perms = std::fs::metadata(path)?.permissions();
10544        perms.set_mode(0o755);
10545        std::fs::set_permissions(path, perms)?;
10546    }
10547    #[cfg(not(unix))]
10548    {
10549        let _ = path;
10550    }
10551    Ok(())
10552}
10553
10554/// What every seat in one batch shares: where the answers are attributed, the
10555/// prompt overlay they inherit, and the build cache they are told to use.
10556///
10557/// A struct rather than four more parameters: `wave` also needs the run's
10558/// state (to record who is answering right now) and the attempt number, and
10559/// eight positional arguments is both unreadable and a clippy error.
10560struct WaveCtx<'a> {
10561    /// Exported as `MAGI_RUN`, so a task an agent files names the run that
10562    /// paid for it.
10563    run: &'a str,
10564    /// Exported as `MAGI_NODE`, and the key the prompt overlay is chosen by.
10565    node: &'a str,
10566    prompts: &'a Prompts,
10567    /// The shared `CARGO_TARGET_DIR`, when the config declares one.
10568    cache: Option<&'a Path>,
10569    /// The review round this wave belongs to, for `"review"`/`"fix"` — see
10570    /// `JobRecord::round`. `None` for every other node.
10571    round: Option<usize>,
10572    /// Carry each seat's failed-agent history across waves (the review loop
10573    /// only): start-of-round priority and handover choice read
10574    /// [`RunState::seat_history`], and every answer or failure writes it.
10575    carry_seats: bool,
10576    /// The run's park / interrupt requests, so a wait for a shared resource
10577    /// can be abandoned the moment the operator asks to park.
10578    pauses: Option<(&'a Pause, &'a Pause)>,
10579}
10580
10581/// Shared scheduling policy for operations and spawned seat invocations.
10582fn repository_wait(
10583    state: &RunState,
10584    pauses: Option<(&Pause, &Pause)>,
10585) -> crate::repo_lock::WaitPolicy {
10586    let pauses = pauses.map(|(a, b)| (a.clone(), b.clone()));
10587    let inherited = crate::repo_lock::WaitPolicy::current();
10588    crate::repo_lock::WaitPolicy::new(
10589        Duration::from_secs(state.config.graph.timeout_resource_wait),
10590        move || {
10591            inherited.cancelled()
10592                || pauses
10593                    .as_ref()
10594                    .is_some_and(|(a, b)| a.parked() || b.parked())
10595        },
10596    )
10597}
10598
10599fn record_repository_block(state: &mut RunState, error: &anyhow::Error) {
10600    if error.is::<crate::repo_lock::ResourceBlocked>() {
10601        state.resource_block = Some(crate::run::ResourceBlock {
10602            cause: crate::run::ResourceBlockCause::Other,
10603            node: "repository".into(),
10604            kind: if error.to_string().contains("disk pressure") {
10605                "disk pressure"
10606            } else if error.to_string().contains("disk admission") {
10607                "disk admission"
10608            } else if error.to_string().contains(crate::incremental::BLOCKED) {
10609                "incremental cache admission"
10610            } else {
10611                "repository coordination"
10612            }
10613            .into(),
10614            detail: error.to_string(),
10615            at: Timestamp::now(),
10616        });
10617        state.event("resource", error.to_string());
10618    }
10619}
10620
10621async fn coordinate_operation(
10622    state: &mut RunState,
10623    cwd: &Path,
10624    wait: &crate::repo_lock::WaitPolicy,
10625) -> Result<Option<crate::repo_lock::SeatLease>> {
10626    let result =
10627        crate::repo_lock::SeatLease::operation_with(cwd, state.config.coordination.as_ref(), wait)
10628            .await;
10629    if let Err(e) = &result {
10630        record_repository_block(state, e);
10631    }
10632    result
10633}
10634
10635/// Run one job, honouring the parallelism budget.
10636async fn run_one(
10637    job: SeatJob,
10638    sem: Arc<Semaphore>,
10639    ctx: &WaveCtx<'_>,
10640    state: &mut RunState,
10641    attempt: usize,
10642) -> (SeatState, AgentOutcome) {
10643    let (_, seat, out) = wave(vec![job], sem, ctx, state, attempt)
10644        .await
10645        .pop()
10646        .expect("one job in, one result out");
10647    (seat, out)
10648}
10649
10650/// Run every job concurrently, capped by the semaphore, preserving order.
10651///
10652/// Every seat in the batch is recorded into [`RunState::active`] before the
10653/// wave starts and cleared as each answer lands, so the run's own record says
10654/// who is still being waited on rather than only who finished.
10655async fn wave(
10656    jobs: Vec<SeatJob>,
10657    sem: Arc<Semaphore>,
10658    ctx: &WaveCtx<'_>,
10659    state: &mut RunState,
10660    attempt: usize,
10661) -> Vec<(usize, SeatState, AgentOutcome)> {
10662    let WaveCtx {
10663        run,
10664        node,
10665        prompts,
10666        cache,
10667        round,
10668        carry_seats: _,
10669        pauses,
10670    } = *ctx;
10671    for job in &jobs {
10672        state.seat_started(node, &job.seat.key, job.timeout, attempt);
10673    }
10674    if let Err(e) = state.save() {
10675        // A failed persist of "who is answering right now" must not abort the
10676        // wave: the seats are already being asked, and the alternative is
10677        // losing the answers to save a status line nobody may even be
10678        // watching.
10679        tracing::warn!("could not persist in-progress seats: {e:#}");
10680    }
10681    // Hold the shared build cache's lease for the whole batch, not per job:
10682    // several candidates (an implement wave) or a fixer legitimately share
10683    // one cache concurrently within this run, and that stays untouched — a
10684    // single lease taken once for the whole wave and released once it is
10685    // done is what stops a *different* borrower (another run's own wave, its
10686    // e2e/gate, a human's `magi review`) from interleaving a build into the
10687    // same directory while this one is in flight. Best-effort, not
10688    // all-or-nothing: a wave that cannot get the lease within its own
10689    // longest job's budget still runs — an hour of paid implementer calls is
10690    // not thrown away over cache contention — but every write-allowed seat
10691    // then goes without `CARGO_TARGET_DIR` for this wave too (see the filter
10692    // below), the same fallback a read-only seat always gets, rather than
10693    // building into a directory this run was never granted. The identity
10694    // record is still invalidated below either way, so the next tracked
10695    // caller (`e2e`/`gate`) never trusts a match it cannot vouch for.
10696    let jobs_had_a_writer = jobs.iter().any(|j| j.allow_write);
10697    // The wait for the lease has its own bound, `timeout_resource_wait`; it is
10698    // never borrowed from a job's agent budget (and never charged to it either,
10699    // see below). The agent's clock starts in `agent::invoke`, at spawn.
10700    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
10701    let cache_guard = if let Some(cache_dir) = cache {
10702        if jobs_had_a_writer {
10703            let owner = crate::cache::Owner::here(run, node, "*", Path::new("(wave)"), "");
10704            let cancel = move || pauses.is_some_and(|(a, b)| a.parked() || b.parked());
10705            match acquire_cache_lease(
10706                state,
10707                cache_dir,
10708                &owner,
10709                resource_limit,
10710                node,
10711                node,
10712                &cancel,
10713            )
10714            .await
10715            {
10716                Ok(Some(g)) => Some(g),
10717                // Parking: leave without spawning anything. The run parks at
10718                // the next boundary and the seats are asked afresh on resume.
10719                Ok(None) => {
10720                    if let Some(block) = &mut state.resource_block {
10721                        block.cause = crate::run::ResourceBlockCause::Cancelled {
10722                            park: pauses.and_then(|(a, b)| b.cause().or_else(|| a.cause())),
10723                        };
10724                    }
10725                    return block_wave(
10726                        jobs,
10727                        state,
10728                        node,
10729                        attempt,
10730                        "parked while waiting for the shared build cache",
10731                    );
10732                }
10733                // An unreadable lease is not contention: keep the old
10734                // fallback (no shared cache for this wave's seats).
10735                Err(e) if e.downcast_ref::<CacheCheckError>().is_some() => None,
10736                Err(e) => {
10737                    // Writing seats may not build into a cache this run does
10738                    // not hold, and an agent that cannot be given one is not
10739                    // asked at all: the run stops resumably instead.
10740                    let why = format!("{e:#}");
10741                    state.event(
10742                        "resource",
10743                        format!("{node}: no agent was started - {why}; resume to ask the same agents again"),
10744                    );
10745                    return block_wave(jobs, state, node, attempt, &why);
10746                }
10747            }
10748        } else {
10749            None
10750        }
10751    } else {
10752        None
10753    };
10754    // Every job keeps its whole budget: the wait above was not the agent's.
10755    // Restart the seats' clocks so "started" means "being spawned now".
10756    state.seats_spawning(node, attempt);
10757    if let Err(e) = state.save() {
10758        tracing::warn!("could not persist the start of a wave: {e:#}");
10759    }
10760    // Queueing behind this wave's own seats for a parallelism slot is normal
10761    // (each of them is bounded by its own agent timeout); only a wait beyond
10762    // all of them plus the resource limit means the slots never came back.
10763    let slot_limit = resource_limit
10764        + jobs
10765            .iter()
10766            .map(|j| j.timeout)
10767            .max()
10768            .unwrap_or_default()
10769            .saturating_mul(jobs.len() as u32);
10770    let mut set = tokio::task::JoinSet::new();
10771    // Seats announce themselves here the moment they hold a parallelism
10772    // permit, i.e. just before their CLI is spawned.
10773    let (spawned_tx, mut spawned_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
10774    let overlay = prompts.overlay(node);
10775    let repository_wait = repository_wait(state, pauses);
10776    for (i, mut job) in jobs.into_iter().enumerate() {
10777        job.prompt = prompt::with_overlay(job.prompt, overlay.clone());
10778        if cache.is_some() {
10779            job.prompt.push('\n');
10780            job.prompt
10781                .push_str(&prompt::build_cache_note(node, job.allow_write));
10782        }
10783        let sem = Arc::clone(&sem);
10784        let run = run.to_owned();
10785        let node = node.to_owned();
10786        // Only implementers were told about the task's attachments, so only
10787        // their seats get the directory widened for reading.
10788        let attachments = if node == "implement" {
10789            state.attachments.clone()
10790        } else {
10791            Vec::new()
10792        };
10793        // A read-only seat is never handed `CARGO_TARGET_DIR` — see
10794        // `prompt::build_cache_note`'s doc for why setting it anyway is
10795        // exactly how a sandboxed reviewer's write refusal got reported as a
10796        // defect in the patch, not a property of its own seat. And a
10797        // write-allowed one is handed it only when the lease above was
10798        // actually acquired: a wave that could not get it (`cache_guard` is
10799        // `None`, see its own comment) must not send seats to build into a
10800        // directory this run does not hold - that is the exact concurrent,
10801        // unmanaged-write race this module exists to prevent, not something
10802        // "proceeding anyway" is allowed to reintroduce.
10803        let cache = cache
10804            .filter(|_| job.allow_write && cache_guard.is_some())
10805            .map(Path::to_path_buf);
10806        let spawned_tx = spawned_tx.clone();
10807        let repository_wait = repository_wait.clone();
10808        let coordination = state.config.coordination.clone();
10809        let fetched_base = state.fetched_base.clone();
10810        set.spawn(async move {
10811            // A wait for a parallelism slot is scheduling, not the agent: it
10812            // is bounded by its own limit and ends as a resource block, never
10813            // as a timeout of the agent that was not started.
10814            let _permit = match tokio::time::timeout(slot_limit, sem.acquire()).await {
10815                Ok(p) => p,
10816                Err(_) => {
10817                    return (
10818                        i,
10819                        job.seat,
10820                        AgentOutcome::ResourceBlocked(format!(
10821                            "no parallel slot became free within {}s",
10822                            slot_limit.as_secs()
10823                        )),
10824                        false,
10825                    );
10826                }
10827            };
10828            spawned_tx.send(job.seat.key.clone()).ok();
10829            let mut seat = job.seat;
10830            let out = repository_wait
10831                .scope(agent::invoke(
10832                    &job.spec,
10833                    &mut seat,
10834                    &Invocation {
10835                        coordination: coordination.as_ref(),
10836                        fetched_base: fetched_base.as_ref(),
10837                        cwd: &job.cwd,
10838                        prompt: &job.prompt,
10839                        timeout: job.timeout,
10840                        allow_write: job.allow_write,
10841                        sessions: job.sessions,
10842                        artifacts: &job.artifacts,
10843                        stem: &job.stem,
10844                        run: &run,
10845                        node: &node,
10846                        cache_dir: cache.as_deref(),
10847                        attachments: &attachments,
10848                        writable: &[],
10849                    },
10850                ))
10851                .await;
10852            let interrupted = out
10853                .as_ref()
10854                .err()
10855                .is_some_and(|error| error.is::<agent::ExecutionInterrupted>());
10856            let out = match out {
10857                Ok(o) if o.quota_exhausted() => AgentOutcome::Quota(o),
10858                Ok(o) if o.usable() => AgentOutcome::Ok(o),
10859                // Billed work the CLI failed to hand over is not an ordinary
10860                // failure, but its text is the CLI's raw error JSON, not an
10861                // answer — `Dropped` keeps it out of `Ok` so a caller cannot
10862                // read it as one by forgetting to check. `usable()` is always
10863                // false here (dropped implies an empty response), so this has
10864                // to be checked before the catch-all `Failed` below or the
10865                // one shape this exists for is lost with the rest.
10866                Ok(o) if o.work_undelivered() => AgentOutcome::Dropped(o),
10867                Ok(o) if o.timed_out => AgentOutcome::Failed(TIMED_OUT.to_owned()),
10868                Ok(o) => AgentOutcome::Failed(format!(
10869                    "process exited with no usable output ({})",
10870                    crate::notices::exit_status(o.exit_code)
10871                )),
10872                Err(e) if e.is::<crate::repo_lock::ResourceBlocked>() => {
10873                    AgentOutcome::ResourceBlocked(e.to_string())
10874                }
10875                Err(e) => AgentOutcome::Failed(e.to_string()),
10876            };
10877            (i, seat, out, interrupted)
10878        });
10879    }
10880    drop(spawned_tx);
10881    let mut collected: Vec<Option<(usize, SeatState, AgentOutcome)>> = Vec::new();
10882    loop {
10883        let joined = tokio::select! {
10884            Some(key) = spawned_rx.recv() => {
10885                // The clock of this seat starts now, not when it queued.
10886                state.seat_spawned(&key);
10887                if let Err(e) = state.save() {
10888                    tracing::warn!("could not persist a seat's start: {e:#}");
10889                }
10890                continue;
10891            }
10892            joined = set.join_next() => joined,
10893        };
10894        let Some(joined) = joined else {
10895            break;
10896        };
10897        let (i, seat, out, interrupted) = match joined {
10898            Ok(v) => v,
10899            // No seat to clear: a panicked task never reported which one it
10900            // was. The defensive sweep below this loop is what stops that
10901            // seat's `active` entry from surviving forever.
10902            Err(e) => {
10903                tracing::error!("agent task panicked: {e}");
10904                continue;
10905            }
10906        };
10907        state.seat_finished(&seat.key);
10908        if interrupted
10909            && node == "implement"
10910            && let Some(label) = state
10911                .candidates
10912                .iter()
10913                .find(|candidate| seat.key == format!("impl-{}", candidate.label))
10914                .map(|candidate| candidate.label)
10915        {
10916            state.candidate_preservation.insert(
10917                label,
10918                "interrupted implementation awaiting persistence".into(),
10919            );
10920            state.interrupted_implementations.insert(label);
10921        }
10922        if let AgentOutcome::ResourceBlocked(why) = &out
10923            && state.resource_block.is_none()
10924        {
10925            state.resource_block = Some(crate::run::ResourceBlock {
10926                cause: crate::run::ResourceBlockCause::Other,
10927                node: node.to_owned(),
10928                kind: if why.contains("disk pressure") {
10929                    "disk pressure"
10930                } else if why.contains("disk admission") {
10931                    "disk admission"
10932                } else if why.contains(crate::incremental::BLOCKED) {
10933                    "incremental cache admission"
10934                } else if why.contains("repository coordination") {
10935                    "repository coordination"
10936                } else {
10937                    "parallel slot"
10938                }
10939                .to_owned(),
10940                detail: why.clone(),
10941                at: Timestamp::now(),
10942            });
10943            let activity = if interrupted {
10944                "agent execution interrupted"
10945            } else {
10946                "no agent was started"
10947            };
10948            state.event("resource", format!("{node}: {activity} - {why}"));
10949        }
10950        record_jobs(state, node, round, &seat.key, &out);
10951        if let Err(e) = state.save() {
10952            tracing::warn!("could not persist a seat's completion: {e:#}");
10953        }
10954        if collected.len() <= i {
10955            collected.resize_with(i + 1, || None);
10956        }
10957        collected[i] = Some((i, seat, out));
10958    }
10959    // Belt-and-braces for the panic branch above: every seat this exact batch
10960    // started shares this `(node, attempt)` pair, and every seat that finished
10961    // normally already cleared itself, so anything left tagged with it here
10962    // can only be a panicked task's leftover. Cleared unconditionally rather
10963    // than left to read as still answering forever.
10964    if state
10965        .active
10966        .values()
10967        .any(|a| a.node == node && a.attempt == attempt)
10968    {
10969        state
10970            .active
10971            .retain(|_, a| !(a.node == node && a.attempt == attempt));
10972        if let Err(e) = state.save() {
10973            tracing::warn!("could not persist the end of a wave: {e:#}");
10974        }
10975    }
10976    // Whether or not the lease above was actually held, several worktrees
10977    // may just have built into the cache with nothing here able to name one
10978    // coherent (worktree, head) for it - see `cache::invalidate_identity`'s
10979    // own doc. Forgetting the old record costs the next `e2e`/`gate` one
10980    // clean it might not have strictly needed; trusting a stale match would
10981    // cost it a wrong answer.
10982    if let Some(cache_dir) = cache
10983        && jobs_had_a_writer
10984    {
10985        crate::cache::invalidate_identity(&crate::run::home(), cache_dir);
10986    }
10987    if let Some(guard) = cache_guard {
10988        guard.release();
10989    }
10990    collected.into_iter().flatten().collect()
10991}
10992
10993/// Fold one seat's [`agent::CommandEvidence`] (if its outcome carries any)
10994/// into the run's [`JobRecord`] log — every node, every seat, uniformly:
10995/// this is data collection, not the fix-specific completion contract in
10996/// [`Runner::continue_fix_report`], and applies regardless of which node
10997/// asked.
10998///
10999/// Only `AgentOutcome::Ok`/`Quota`/`Dropped` carry an [`AgentOutput`] to read
11000/// evidence from; `Failed` does not, and correctly contributes nothing — a
11001/// timeout or crash is not itself evidence about a command the seat may have
11002/// started.
11003fn record_jobs(
11004    state: &mut RunState,
11005    node: &str,
11006    round: Option<usize>,
11007    seat: &str,
11008    out: &AgentOutcome,
11009) {
11010    let commands: &[agent::CommandEvidence] = match out {
11011        AgentOutcome::Ok(o) | AgentOutcome::Quota(o) | AgentOutcome::Dropped(o) => &o.commands,
11012        AgentOutcome::Failed(_) | AgentOutcome::ResourceBlocked(_) => &[],
11013    };
11014    let checked_at = Timestamp::now();
11015    for c in commands {
11016        state.jobs.push(JobRecord {
11017            node: node.to_owned(),
11018            round,
11019            seat: seat.to_owned(),
11020            id: c.id.clone(),
11021            description: c.description.clone(),
11022            checked_at,
11023            status: match c.exit_code {
11024                Some(0) => JobStatus::Completed,
11025                Some(_) => JobStatus::Failed,
11026                None => JobStatus::Unknown,
11027            },
11028            exit_code: c.exit_code,
11029            result_summary: c.result_summary.clone(),
11030            source: c.source.clone(),
11031        });
11032    }
11033}
11034
11035/// Is a review round clean, given how many reviewer seats answered against
11036/// how many the round expected?
11037///
11038/// A seat that never answered (timeout, crash, unparsable output) is not a
11039/// seat that read the patch and found nothing — treating it as such is
11040/// exactly the bug this function exists to close. Under the default `block`
11041/// policy a missing seat can never be clean; `warn` still requires the seats
11042/// that *did* answer to have found nothing blocking and verification to be
11043/// green.
11044///
11045/// `quota_missing` narrows that `block` default for exactly one cause of
11046/// absence: a seat lost to its own rate limit this round. Re-reviewing hoping
11047/// a session limit lifts by the very next round buys nothing — the seat is
11048/// asked again with the same quota — so once every missing seat is accounted
11049/// for by a quota loss (and at least one seat *did* answer, so a decision has
11050/// something to rest on) the round is decided on the panel that could answer,
11051/// same as `warn` would. A panel that lost every seat to quota is not
11052/// decided here: `answered == 0` falls through to the existing `block`
11053/// fallback so a fully collapsed panel still waits rather than landing on no
11054/// review at all.
11055fn round_is_clean(
11056    blocking: usize,
11057    e2e_ok: bool,
11058    answered: usize,
11059    expected: usize,
11060    quota_missing: usize,
11061    policy: IncompleteReviewPolicy,
11062) -> bool {
11063    if blocking != 0 || !e2e_ok {
11064        return false;
11065    }
11066    if answered == expected || policy == IncompleteReviewPolicy::Warn {
11067        return true;
11068    }
11069    answered > 0 && expected - answered <= quota_missing
11070}
11071
11072/// The review loop's own conclusion, derived entirely from its persisted
11073/// round records and the round budget that produced them — never from
11074/// `status`, so a reentry (or `gate`/`merge` reading it independently)
11075/// recomputes the identical answer regardless of what an earlier node in the
11076/// same walk, or a previous walk, did to `status`.
11077///
11078/// `None` while more rounds remain to try, including when review never ran
11079/// at all (`review_rounds = 0`, or nothing yet recorded). Once a round has
11080/// gone clean, or the budget is spent, or the tree has stopped moving (see
11081/// [`STAGNANT_LIMIT`]), the answer is one of two things:
11082///
11083/// - An incomplete panel that raised nothing is missing input, not a
11084///   verified tree — never a hand-off candidate, whatever verification said
11085///   (see [`ReviewRound::incomplete`], `IncompleteReviewPolicy`).
11086/// - Otherwise, green e2e on the last round hands off (see
11087///   [`Runner::stop_reviewing`]); red e2e blocks.
11088///
11089/// A last round whose own verification is still `ResourceBlocked` — magi
11090/// itself never got a command to run, not evidence the patch is broken —
11091/// is neither: this returns `None` for it too, the same as "more rounds
11092/// remain", so a reentry retries the check (see `Runner::review_loop`'s own
11093/// handling of that shape) instead of this cheap recomputation guessing a
11094/// verdict a real attempt never produced.
11095pub(crate) fn review_conclusion(reviews: &[ReviewRound], max_rounds: usize) -> Option<RunStatus> {
11096    if max_rounds == 0
11097        || reviews.last().is_some_and(|r| {
11098            r.clean && matches!(r.e2e_status(), E2eStatus::Passed | E2eStatus::NotConfigured)
11099        })
11100    {
11101        return Some(RunStatus::Gating);
11102    }
11103    let last = reviews.last()?;
11104    let stagnant = reviews.iter().rev().take_while(|r| !r.progressed).count() >= STAGNANT_LIMIT;
11105    if reviews.len() < max_rounds && !stagnant {
11106        return None;
11107    }
11108    if last.incomplete() && last.blocking == 0 {
11109        return Some(RunStatus::Blocked);
11110    }
11111    if last.e2e_status() == E2eStatus::ResourceBlocked
11112        || (last.e2e_status() == E2eStatus::Deferred && last.blocking == 0 && last.fix.is_none())
11113    {
11114        return None;
11115    }
11116    Some(if last.e2e.iter().all(CommandOutcome::ok) {
11117        RunStatus::Gating
11118    } else {
11119        RunStatus::Blocked
11120    })
11121}
11122
11123/// How long a re-ask may take, given the budget the first attempt had.
11124///
11125/// A `nudged` retry is a request to restate an answer the seat has already
11126/// worked out: it carries no new work, so it does not deserve the original
11127/// budget. Measured on run 01c2, two judges restated their ranking in 41 and
11128/// 133 seconds while a third sat for over ten minutes on a resumed session
11129/// holding 410 KB of prior output - and because the retry had inherited the
11130/// full 1200s judge timeout, one stuck nudge nearly doubled the wall time of a
11131/// judging round whose other seats were long finished.
11132///
11133/// A quarter of the budget, with a floor so that a deliberately short timeout
11134/// does not collapse to nothing. A retry that re-sends the whole prompt
11135/// (because the seat kept no context) is the original job again, and keeps the
11136/// original budget.
11137pub(crate) fn retry_budget(full: Duration, nudged: bool) -> Duration {
11138    if nudged {
11139        (full / 4).max(Duration::from_secs(120)).min(full)
11140    } else {
11141        full
11142    }
11143}
11144
11145/// Run a wave and parse each reply, re-asking the seats whose reply was
11146/// unusable.
11147///
11148/// The re-ask is a nudge rather than the whole prompt again when the seat still
11149/// holds its conversation, which is the difference between a cheap retry and
11150/// paying for the entire candidate set twice.
11151///
11152/// A seat whose agent *fails* (rate limit, timeout, any other error) and has a
11153/// successor in `roster` is handed to it instead of being re-asked: the
11154/// handover is the retry. A seat with no successor left (a single-agent
11155/// roster, the roster's tail) is nudged as before, up to `retries` times. So
11156/// the asks to one seat in one node number at most
11157/// `roster.len().max(1) * (1 + retries)`; an agent that still has a successor
11158/// is asked once (a dropped stream is nudged first), and only the last agent
11159/// of the chain gets the `retries` same-agent nudges. Each roster agent is
11160/// tried at most once per seat, walking forward from the seat's own position and never wrapping
11161/// ([`next_untried_in_roster`]); a quota or timeout always hands over, any
11162/// other failure stops the chain when the previous agent failed the same way
11163/// ([`should_hand_over`]). The new agent takes a fresh [`SeatState`], so
11164/// [`has_context`] is false and the job's own full prompt and full budget are
11165/// sent. A seat whose chain ends on a quota records one [`QuotaLoss`] (the
11166/// intermediate ones are not losses) and is returned as a failure like any
11167/// other absent seat — the caller decides whether the panel still has a
11168/// quorum. An empty `roster` disables handover: failures are nudged as they
11169/// always were, and a quota is simply lost. A reply that fails to parse or
11170/// validate is the prompt's doing and is only ever nudged, never handed over.
11171///
11172/// The returned [`SeatState`] names the agent that answered (or tried last).
11173#[allow(clippy::too_many_arguments)]
11174async fn ask_json_wave<T>(
11175    jobs: Vec<SeatJob>,
11176    sem: Arc<Semaphore>,
11177    retries: usize,
11178    roster: &[AgentSpec],
11179    ctx: &WaveCtx<'_>,
11180    losses: &mut Vec<QuotaLoss>,
11181    state: &mut RunState,
11182    validate: &(dyn Fn(&T) -> Result<()> + Send + Sync),
11183) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11184where
11185    T: serde::de::DeserializeOwned + Send + 'static,
11186{
11187    ask_wave_with(
11188        jobs,
11189        sem,
11190        retries,
11191        roster,
11192        ctx,
11193        losses,
11194        state,
11195        &|text: &str| {
11196            let v = verdict::extract_json::<T>(text)?;
11197            validate(&v)?;
11198            Ok(v)
11199        },
11200    )
11201    .await
11202}
11203
11204/// [`ask_json_wave`] with the reading of an answer supplied by the caller, so
11205/// a node whose answer is prose (deliberation) shares the same handover,
11206/// failure classification, quota bookkeeping and bounds instead of a copy.
11207///
11208/// A seat handed to another roster agent is sent the job's `handover` prompt
11209/// (when it has one) rather than `prompt`: the new agent has no session, so a
11210/// resume-style prompt would be incomplete. That holds for the handover ask
11211/// and for every nudge to that agent whose `has_context` is false.
11212#[allow(clippy::too_many_arguments)]
11213async fn ask_wave_with<T>(
11214    jobs: Vec<SeatJob>,
11215    sem: Arc<Semaphore>,
11216    retries: usize,
11217    roster: &[AgentSpec],
11218    ctx: &WaveCtx<'_>,
11219    losses: &mut Vec<QuotaLoss>,
11220    state: &mut RunState,
11221    parse: &(dyn Fn(&str) -> Result<T> + Send + Sync),
11222) -> Vec<(SeatState, Result<(T, AgentOutput)>, usize)>
11223where
11224    T: Send + 'static,
11225{
11226    let n = jobs.len();
11227    let originals: Vec<SeatJob> = jobs;
11228    let mut seats: Vec<SeatState> = originals.iter().map(|j| j.seat.clone()).collect();
11229    let mut done: Vec<Option<Result<(T, AgentOutput)>>> = (0..n).map(|_| None).collect();
11230    // Nudges each seat's *current* agent has taken — 0 for a first-ask
11231    // answer, N once it has gone through N nudges. Read back once this
11232    // returns, so a caller building a history record (`ReviewRecord`) can
11233    // tell "never answered" (`failed: Some(_)`, `attempts == 0`) apart from
11234    // "recovered after a nudge" (`failed: None`, `attempts > 0`) — see that
11235    // field's own doc.
11236    let mut nudges: Vec<usize> = vec![0; n];
11237    // The agent now occupying each seat, the ids it has already been through,
11238    // where in the roster the walk began, the class of the last failure, and
11239    // the stem word of a handover not yet asked (full prompt, full budget).
11240    let mut specs: Vec<AgentSpec> = originals.iter().map(|j| j.spec.clone()).collect();
11241    let mut tried: Vec<BTreeSet<String>> = specs
11242        .iter()
11243        .map(|s| BTreeSet::from([s.id.clone()]))
11244        .collect();
11245    let starts: Vec<usize> = specs
11246        .iter()
11247        .map(|s| roster.iter().position(|r| r.id == s.id).unwrap_or(0))
11248        .collect();
11249    let carry = ctx.carry_seats && !roster.is_empty();
11250    // Carried across rounds: ids that failed the seat earlier, and how the
11251    // last failure went (so a repeat of it is not handed over again).
11252    let carried: Vec<BTreeSet<String>> = originals
11253        .iter()
11254        .map(|j| {
11255            state
11256                .seat_history
11257                .get(&j.seat.key)
11258                .filter(|_| carry)
11259                .map(|h| h.failed.clone())
11260                .unwrap_or_default()
11261        })
11262        .collect();
11263    let mut prev: Vec<Option<FailClass>> = originals
11264        .iter()
11265        .map(|j| {
11266            state
11267                .seat_history
11268                .get(&j.seat.key)
11269                .filter(|_| carry)
11270                .and_then(|h| h.last_fail.clone())
11271        })
11272        .collect();
11273    let next_agent =
11274        |i: usize, tried: &BTreeSet<String>, specs: &[AgentSpec]| -> Option<AgentSpec> {
11275            let others: BTreeSet<String> = specs
11276                .iter()
11277                .enumerate()
11278                .filter(|(j, _)| *j != i)
11279                .map(|(_, s)| s.id.clone())
11280                .collect();
11281            pick_successor(
11282                roster,
11283                starts[i],
11284                tried,
11285                carry.then(|| &carried[i]),
11286                &others,
11287            )
11288            .cloned()
11289        };
11290    let mut fresh: Vec<Option<String>> = vec![None; n];
11291    let mut last_quota: Vec<Option<agent::Quota>> = vec![None; n];
11292    let mut pending: Vec<usize> = (0..n).collect();
11293
11294    // Per seat the work is bounded by `roster.len().max(1) * (1 + retries)`
11295    // asks: an agent with a successor is asked once and handed over, and only
11296    // a seat with no successor spends `retries` nudges on the same agent. This
11297    // only guarantees the loop's own termination whatever those say.
11298    let max_rounds = (retries + 1) * roster.len().max(1) + 1;
11299    for round in 0..max_rounds {
11300        if pending.is_empty() {
11301            break;
11302        }
11303        let mut batch = Vec::with_capacity(pending.len());
11304        let mut renudged: Vec<&str> = Vec::new();
11305        for &i in &pending {
11306            let src = &originals[i];
11307            // A seat now held by another agent than the job named has no
11308            // session of its own: it gets the full-context prompt whenever
11309            // it is asked in full (the handover ask, a nudge it cannot
11310            // resume).
11311            let full: &str = match &src.handover {
11312                Some(h) if specs[i].id != src.spec.id => h,
11313                _ => &src.prompt,
11314            };
11315            // The prompt and the budget are one decision: a nudge restates
11316            // finished work, a re-sent prompt redoes it.
11317            let (prompt, timeout, stem) = if let Some(word) = fresh[i].take() {
11318                (
11319                    full.to_owned(),
11320                    src.timeout,
11321                    format!("{}-{word}-{}", src.stem, specs[i].id),
11322                )
11323            } else if nudges[i] == 0 {
11324                (full.to_owned(), src.timeout, src.stem.clone())
11325            } else {
11326                renudged.push(src.seat.key.as_str());
11327                let why = done[i]
11328                    .as_ref()
11329                    .and_then(|r| r.as_ref().err().map(ToString::to_string))
11330                    .unwrap_or_else(|| "no parsable answer".to_owned());
11331                let nudge = prompt::nudge(&why);
11332                let nudged = has_context(&specs[i], &seats[i], src.sessions);
11333                let prompt = if nudged {
11334                    nudge
11335                } else {
11336                    format!("{full}\n\n---\n\n{nudge}")
11337                };
11338                (
11339                    prompt,
11340                    retry_budget(src.timeout, nudged),
11341                    format!("{}-retry{}", src.stem, nudges[i]),
11342                )
11343            };
11344            batch.push(SeatJob {
11345                spec: specs[i].clone(),
11346                seat: seats[i].clone(),
11347                cwd: src.cwd.clone(),
11348                prompt,
11349                timeout,
11350                allow_write: src.allow_write,
11351                sessions: src.sessions,
11352                artifacts: src.artifacts.clone(),
11353                stem,
11354                handover: None,
11355            });
11356        }
11357
11358        if !renudged.is_empty() {
11359            state.event(
11360                ctx.node,
11361                format!("retry {round}: re-asking {}", renudged.join(", ")),
11362            );
11363        }
11364        let results = wave(batch, Arc::clone(&sem), ctx, state, round).await;
11365        let mut still = Vec::new();
11366        for (&i, (_wi, seat, out)) in pending.iter().zip(results) {
11367            seats[i] = seat;
11368            // No agent was asked: nothing to hand over, nudge, remember or
11369            // charge. The caller sees `resource_block` and stops the node.
11370            if let AgentOutcome::ResourceBlocked(why) = &out {
11371                done[i] = Some(Err(anyhow::anyhow!("{why}")));
11372                continue;
11373            }
11374            let class = FailClass::of(&out);
11375            // A dropped stream is nudged first (the conversation is still
11376            // there to pick up); only a seat whose nudges are spent hands over.
11377            let nudge_first = matches!(out, AgentOutcome::Dropped(_))
11378                && nudges[i] < retries
11379                && !roster.is_empty();
11380            if let Some(cur) = class.clone().filter(|_| !roster.is_empty() && !nudge_first) {
11381                let next = should_hand_over(prev[i].as_ref(), &cur)
11382                    .then(|| next_agent(i, &tried[i], &specs))
11383                    .flatten();
11384                if carry {
11385                    let h = state
11386                        .seat_history
11387                        .entry(originals[i].seat.key.clone())
11388                        .or_default();
11389                    h.failed.insert(specs[i].id.clone());
11390                    h.last_fail = Some(cur.clone());
11391                    if h.last_ok.as_deref() == Some(specs[i].id.as_str()) {
11392                        h.last_ok = None;
11393                    }
11394                    // Saved before the next agent is asked, so a restart in
11395                    // between does not forget who failed.
11396                    if let Err(e) = state.save() {
11397                        tracing::warn!("could not persist a seat's failure history: {e:#}");
11398                    }
11399                }
11400                if let Some(next) = next {
11401                    record_handover(
11402                        state,
11403                        ctx.node,
11404                        &originals[i].seat.key,
11405                        &specs[i].id,
11406                        &next.id,
11407                        &cur,
11408                        &fail_reason(&out),
11409                    );
11410                    tried[i].insert(next.id.clone());
11411                    prev[i] = Some(cur.clone());
11412                    seats[i] =
11413                        handover_seat(&originals[i].seat.key, &next.id, state.next_seat_seed());
11414                    specs[i] = next;
11415                    fresh[i] = Some(cur.stem_word().to_owned());
11416                    nudges[i] = 0;
11417                    done[i] = Some(Err(anyhow::anyhow!(
11418                        "handed over after: {}",
11419                        fail_reason(&out)
11420                    )));
11421                    still.push(i);
11422                    continue;
11423                }
11424            }
11425            if carry
11426                && !nudge_first
11427                && let Some(cur) = class.clone()
11428            {
11429                // The chain ended here (no successor, or a repeated failure).
11430                let h = state
11431                    .seat_history
11432                    .entry(originals[i].seat.key.clone())
11433                    .or_default();
11434                h.failed.insert(specs[i].id.clone());
11435                h.last_fail = Some(cur);
11436            }
11437            let parsed = match out {
11438                AgentOutcome::Ok(o) => parse(&o.text).map(|v| (v, o)),
11439                AgentOutcome::Quota(o) => {
11440                    last_quota[i] = o.quota.clone();
11441                    Err(anyhow::anyhow!("rate limited (quota); not retrying now"))
11442                }
11443                // Not a parseable answer: the nudge loop re-asks it, which is
11444                // exactly what a dropped stream needs. Just don't hand its raw
11445                // error JSON to `extract_json`.
11446                AgentOutcome::Dropped(o) => {
11447                    let why = o
11448                        .dropped
11449                        .as_ref()
11450                        .map(|d| d.why.as_str())
11451                        .unwrap_or("the CLI ended the stream without delivering its answer");
11452                    Err(anyhow::anyhow!("the CLI dropped the stream ({why})"))
11453                }
11454                AgentOutcome::Failed(e) | AgentOutcome::ResourceBlocked(e) => {
11455                    Err(anyhow::anyhow!(e))
11456                }
11457            };
11458            let quota = class == Some(FailClass::Quota);
11459            let failed = parsed.is_err();
11460            if !failed && matches!(ctx.node, "judge" | "vote") {
11461                // A validated answer restores this seat's quorum presence,
11462                // including on the normal path after an all-seat block.
11463                state
11464                    .quota
11465                    .retain(|loss| loss.seat != originals[i].seat.key);
11466            }
11467            done[i] = Some(parsed);
11468            if carry && !failed {
11469                let h = state
11470                    .seat_history
11471                    .entry(originals[i].seat.key.clone())
11472                    .or_default();
11473                h.failed.remove(&specs[i].id);
11474                h.last_ok = Some(specs[i].id.clone());
11475                h.last_fail = None;
11476            }
11477            // Do not re-ask a rate-limited seat (quota) — a retry is known to
11478            // fail the same way; and never re-ask a seat that already parsed.
11479            // A failed agent that still has a successor is not re-asked
11480            // either: the handover was its remedy and has just been refused
11481            // (the chain stops on a repeated failure class). A seat with no
11482            // successor left (a single-agent roster, the roster's tail, or an
11483            // empty roster) keeps the same-agent nudge, bounded by `retries`.
11484            let agent_failure = class.is_some()
11485                && !nudge_first
11486                && !roster.is_empty()
11487                && next_agent(i, &tried[i], &specs).is_some();
11488            if failed && !quota && !agent_failure && nudges[i] < retries {
11489                nudges[i] += 1;
11490                still.push(i);
11491            }
11492        }
11493        pending = still;
11494    }
11495
11496    // One loss per seat whose chain ended on a quota: the intermediate ones
11497    // were absorbed by a handover and are not losses.
11498    for (i, q) in last_quota.into_iter().enumerate() {
11499        if let Some(quota) = q {
11500            losses.push(QuotaLoss::from_report(
11501                originals[i].seat.key.clone(),
11502                ctx.node,
11503                Some(&quota),
11504            ));
11505        }
11506    }
11507
11508    // A stalled quorum already has its own recovery checkpoint. Refresh
11509    // its missing seats before deciding whether it can resume.
11510    if state.status != RunStatus::Stalled
11511        && ctx.node != "deliberate"
11512        && n > 0
11513        && losses.len() == n
11514        && done.iter().all(|r| r.as_ref().is_some_and(Result::is_err))
11515    {
11516        block_machine_outage(state, ctx.node, losses);
11517        state.quota.append(losses);
11518    }
11519
11520    seats
11521        .into_iter()
11522        .zip(done)
11523        .zip(nudges)
11524        .map(|((seat, res), attempts)| {
11525            (
11526                seat,
11527                res.unwrap_or_else(|| Err(anyhow::anyhow!("no attempt was made"))),
11528                attempts,
11529            )
11530        })
11531        .collect()
11532}
11533
11534/// A node with no available agent has no task verdict. Preserve the node
11535/// checkpoint and record why a subsequent explicit resume is needed.
11536pub(crate) fn block_machine_outage(state: &mut RunState, node: &str, losses: &[QuotaLoss]) {
11537    let mut auth: Vec<_> = losses
11538        .iter()
11539        .filter(|loss| loss.cause == agent::QuotaCause::Authentication)
11540        .map(|loss| loss.agent.clone())
11541        .collect();
11542    auth.sort();
11543    auth.dedup();
11544    let (kind, detail) = if auth.is_empty() {
11545        (
11546            "agent usage limit",
11547            "All seats reached their agent usage limits; resume after usage is available"
11548                .to_owned(),
11549        )
11550    } else {
11551        (
11552            "agent authentication",
11553            format!(
11554                "Agents {} cannot authenticate; re-authenticate these agents, then resume the held task",
11555                auth.join(", ")
11556            ),
11557        )
11558    };
11559    state.resource_block = Some(crate::run::ResourceBlock {
11560        node: node.to_owned(),
11561        kind: kind.to_owned(),
11562        cause: crate::run::ResourceBlockCause::Other,
11563        detail: detail.clone(),
11564        at: Timestamp::now(),
11565    });
11566    state.event(node, detail);
11567}
11568
11569/// Did the wave end without asking any agent (see [`block_wave`])?
11570fn wave_blocked(results: &[(usize, SeatState, AgentOutcome)]) -> bool {
11571    results
11572        .iter()
11573        .any(|(_, _, o)| matches!(o, AgentOutcome::ResourceBlocked(_)))
11574}
11575
11576/// Settle a wave that never spawned anything: every job answers
11577/// [`AgentOutcome::ResourceBlocked`], its seat leaves `active`, and nothing
11578/// else about the seat (session, history, quota) is touched.
11579fn block_wave(
11580    jobs: Vec<SeatJob>,
11581    state: &mut RunState,
11582    node: &str,
11583    attempt: usize,
11584    why: &str,
11585) -> Vec<(usize, SeatState, AgentOutcome)> {
11586    state.resource_wait = None;
11587    state
11588        .active
11589        .retain(|_, a| !(a.node == node && a.attempt == attempt));
11590    if let Err(e) = state.save() {
11591        tracing::warn!("could not persist a resource-blocked wave: {e:#}");
11592    }
11593    jobs.into_iter()
11594        .enumerate()
11595        .map(|(i, j)| (i, j.seat, AgentOutcome::ResourceBlocked(why.to_owned())))
11596        .collect()
11597}
11598
11599/// Acquire the shared build cache's lease, waiting out contention within
11600/// `budget` (never past it — see AGENTS.md's build-cache section on why an
11601/// unbounded wait is never acceptable).
11602///
11603/// A first, non-blocking check happens before ever waiting; if it finds the
11604/// lease busy, that fact is logged as a `verify` event *and* flushed with
11605/// [`RunState::save`] immediately — not only once the wait finally succeeds
11606/// or gives up — so a `magi show` run by a different process while this one
11607/// is still waiting reads a `run.json` that says so, rather than whatever it
11608/// looked like before the wait started. The same applies to the terminal
11609/// failure: logged and saved before this returns `Err`, so a caller that
11610/// could not get the lease at all still leaves a legible record of why.
11611async fn acquire_cache_lease(
11612    state: &mut RunState,
11613    cache_dir: &Path,
11614    owner: &crate::cache::Owner,
11615    budget: Duration,
11616    node: &str,
11617    context: &str,
11618    cancel: &(dyn Fn() -> bool + Sync),
11619) -> Result<Option<crate::cache::Guard>> {
11620    let home = crate::run::home();
11621    let started = Instant::now();
11622    let busy = match crate::cache::try_acquire(&home, cache_dir, owner) {
11623        Ok(crate::cache::AcquireOutcome::Acquired(g)) => return Ok(Some(g)),
11624        Ok(crate::cache::AcquireOutcome::Busy(busy)) => busy,
11625        Err(e) => {
11626            state.event(
11627                "verify",
11628                format!("{context}: could not check the shared build cache: {e:#}"),
11629            );
11630            if let Err(e2) = state.save() {
11631                tracing::warn!("could not persist a cache-check failure: {e2:#}");
11632            }
11633            return Err(anyhow::Error::new(CacheCheckError(format!("{e:#}"))));
11634        }
11635    };
11636    state.event(
11637        "verify",
11638        format!(
11639            "{context}: waiting for the shared build cache at {} ({})",
11640            cache_dir.display(),
11641            busy.describe()
11642        ),
11643    );
11644    // Seats of this node are waiting, not running: say so, so the countdown a
11645    // reader sees only ever measures an agent that has been spawned.
11646    state.resource_wait = Some(crate::run::ResourceWait {
11647        node: context.to_owned(),
11648        kind: "shared build cache".to_owned(),
11649        holder: busy.describe(),
11650        since: Timestamp::now(),
11651        limit_secs: budget.as_secs(),
11652    });
11653    if let Err(e) = state.save() {
11654        tracing::warn!("could not persist a cache wait: {e:#}");
11655    }
11656    let remaining = budget.saturating_sub(started.elapsed());
11657    let waited = crate::cache::wait_for_unless(
11658        &home,
11659        cache_dir,
11660        owner,
11661        remaining,
11662        Duration::from_secs(5),
11663        cancel,
11664    )
11665    .await;
11666    state.resource_wait = None;
11667    let cause = match &waited {
11668        Ok(Some(_)) => None,
11669        Ok(None) => Some(crate::run::ResourceBlockCause::Cancelled { park: None }),
11670        Err(error) if error.is::<crate::cache::WaitTimeout>() => {
11671            Some(crate::run::ResourceBlockCause::Timeout {
11672                waited_secs: started.elapsed().as_secs(),
11673                limit_secs: budget.as_secs(),
11674            })
11675        }
11676        Err(_) => Some(crate::run::ResourceBlockCause::Other),
11677    };
11678    if let Some(cause) = cause {
11679        let detail = match &waited {
11680            Ok(_) => "the wait for the build cache was cancelled".to_owned(),
11681            Err(error) => format!("{error:#}"),
11682        };
11683        state.resource_block = Some(crate::run::ResourceBlock {
11684            cause,
11685            node: node.to_owned(),
11686            kind: "shared build cache".into(),
11687            detail,
11688            at: Timestamp::now(),
11689        });
11690        state.event(
11691            "resource",
11692            state.resource_block.as_ref().unwrap().description(),
11693        );
11694    }
11695    if let Err(error) = state.save() {
11696        tracing::warn!("could not persist cache acquisition outcome: {error:#}");
11697    }
11698    waited
11699}
11700
11701/// The lease state could not be read at all (unreadable lease file, a
11702/// permission problem). Unlike a busy lease that outlasts its bound, nothing is
11703/// known to be contended, so a wave degrades to running without a shared cache
11704/// instead of stopping on a hold that [`crate::triage`] could never release.
11705#[derive(Debug)]
11706struct CacheCheckError(String);
11707
11708impl std::fmt::Display for CacheCheckError {
11709    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
11710        f.write_str(&self.0)
11711    }
11712}
11713
11714impl std::error::Error for CacheCheckError {}
11715
11716/// Keep a specific acquisition outcome when the caller adds command evidence.
11717fn record_cache_error(state: &mut RunState, node: &str, error: &anyhow::Error) {
11718    state
11719        .resource_block
11720        .get_or_insert_with(|| crate::run::ResourceBlock {
11721            cause: crate::run::ResourceBlockCause::Other,
11722            node: node.to_owned(),
11723            kind: "shared build cache".into(),
11724            detail: format!("{error:#}"),
11725            at: Timestamp::now(),
11726        });
11727    if let Err(error) = state.save() {
11728        tracing::warn!("could not persist cache resource block: {error:#}");
11729    }
11730}
11731
11732/// Run `body` — a verify command batch — while holding the shared build
11733/// cache's lease, so this run's own full verification (`e2e`, `gate`) can
11734/// never interleave with another borrower's build against the same
11735/// `CARGO_TARGET_DIR`: a different run, a lingering reviewer past its
11736/// timeout, or a human's own `magi review`. See the `cache` module doc for
11737/// why this matters more than Cargo's own per-target locking covers — two
11738/// *different* worktrees building the same package name/version into one
11739/// cache directory is a staleness bug, not a lock contention one.
11740///
11741/// Cache acquisition has its own bound, `timeout_resource_wait`, just as
11742/// agent waves do. Repository scheduling also has its own bounded,
11743/// cancellable resource-wait policy. Contention must not consume the
11744/// verification budget: `body` receives the full `budget` after both leases
11745/// are acquired and the cache is fresh. Without a shared cache, only
11746/// repository coordination is acquired before handing over the full budget.
11747///
11748/// A lease that cannot be acquired within the resource wait is reported as a single
11749/// synthetic [`CommandOutcome`] (`code: None`) rather than silently skipping
11750/// verification — the same shape a spawn failure already takes in
11751/// [`run_commands`], so a caller need not special-case it.
11752#[allow(clippy::too_many_arguments)]
11753async fn with_cache_lease<'s, F, Fut>(
11754    state: &'s mut RunState,
11755    cache_dir: Option<&Path>,
11756    node: &str,
11757    seat: &str,
11758    worktree: &Path,
11759    head: &str,
11760    budget: Duration,
11761    context: &str,
11762    wait: &crate::repo_lock::WaitPolicy,
11763    body: F,
11764) -> Result<(Vec<CommandOutcome>, bool)>
11765where
11766    F: FnOnce(&'s mut RunState, Duration) -> Fut,
11767    Fut: std::future::Future<Output = (Vec<CommandOutcome>, bool, Vec<u32>)>,
11768{
11769    // Match editing waves: cache first, repository second. In particular,
11770    // pre_gate must never hold the repository while waiting for a wave's cache.
11771    let started = Instant::now();
11772    let home = crate::run::home();
11773    let resource_limit = Duration::from_secs(state.config.graph.timeout_resource_wait);
11774    let guard = if let Some(cache_dir) = cache_dir {
11775        let owner = crate::cache::Owner::here(&state.id, node, seat, worktree, head);
11776        match acquire_cache_lease(
11777            state,
11778            cache_dir,
11779            &owner,
11780            resource_limit,
11781            node,
11782            context,
11783            &|| wait.cancelled(),
11784        )
11785        .await
11786        .and_then(|g| {
11787            g.ok_or_else(|| anyhow::anyhow!("the wait for the build cache was cancelled"))
11788        }) {
11789            Ok(g) => Some(g),
11790            Err(e) => {
11791                record_cache_error(state, node, &e);
11792                return Ok((
11793                    vec![CommandOutcome {
11794                        command: "(waiting for the shared build cache)".into(),
11795                        code: None,
11796                        output_tail: e.to_string(),
11797                        duration_ms: started.elapsed().as_millis() as u64,
11798                        resource_blocked: true,
11799                    }],
11800                    false,
11801                ));
11802            }
11803        }
11804    } else {
11805        None
11806    };
11807    let _coordination = coordinate_operation(state, worktree, wait).await?;
11808    if let Some(cache_dir) = cache_dir {
11809        let identity = crate::cache::Identity::new(worktree, head);
11810        if let Err(e) = crate::cache::ensure_fresh(
11811            &home,
11812            cache_dir,
11813            &identity,
11814            &state.config.verify.cargo_manifests(),
11815        ) {
11816            record_cache_error(state, node, &e);
11817            state.event(
11818                "verify",
11819                format!(
11820                    "{context}: could not confirm the shared build cache matches {} at {}: {e:#}",
11821                    worktree.display(),
11822                    short(head)
11823                ),
11824            );
11825            if let Some(guard) = guard {
11826                guard.release();
11827            }
11828            return Ok((
11829                vec![CommandOutcome {
11830                    command: "(confirming the shared build cache is fresh)".into(),
11831                    code: None,
11832                    output_tail: e.to_string(),
11833                    duration_ms: started.elapsed().as_millis() as u64,
11834                    resource_blocked: true,
11835                }],
11836                false,
11837            ));
11838        }
11839    }
11840    let (outcomes, retried, timed_out_pids) = body(state, budget).await;
11841    if !timed_out_pids.is_empty() {
11842        wait_for_timed_out_children_to_die(&timed_out_pids).await;
11843    }
11844    // Drop repository coordination before releasing the cache, reversing the
11845    // acquisition order. A cancelled/error future also drops both guards.
11846    drop(_coordination);
11847    if let Some(guard) = guard {
11848        guard.release();
11849    }
11850    Ok((outcomes, retried))
11851}
11852
11853/// Poll `pids` — commands [`run_commands`] reports as still running when its
11854/// own timeout elapsed — until every one is confirmed gone, or
11855/// [`LEASE_RELEASE_MAX_WAIT`] passes, whichever comes first.
11856///
11857/// Real confirmation where confirmation is possible, not a substitute for
11858/// full process-tree observation: a grandchild the timed-out process spawned
11859/// and that survives independently of it is invisible to a pid check the
11860/// same way it always was, and continuing to observe and collect *that*
11861/// stays a different piece of work with its own owner. This only narrows a
11862/// fixed blind wait into an actual check of the pids this process does know
11863/// about.
11864async fn wait_for_timed_out_children_to_die(pids: &[u32]) {
11865    wait_for_pids_with(
11866        pids,
11867        crate::proc::pid_alive,
11868        LEASE_RELEASE_POLL,
11869        LEASE_RELEASE_MAX_WAIT,
11870    )
11871    .await;
11872}
11873
11874/// [`wait_for_timed_out_children_to_die`] with its liveness query, poll
11875/// interval and ceiling supplied by the caller, so the polling *logic* -
11876/// returns as soon as every pid reports dead, gives up at the ceiling
11877/// otherwise - is testable on millisecond durations without asking the real
11878/// OS about a pid at all.
11879async fn wait_for_pids_with<F: Fn(u32) -> bool>(
11880    pids: &[u32],
11881    alive: F,
11882    poll: Duration,
11883    max_wait: Duration,
11884) {
11885    let deadline = Instant::now() + max_wait;
11886    loop {
11887        if pids.iter().all(|&pid| !alive(pid)) {
11888            return;
11889        }
11890        if Instant::now() >= deadline {
11891            return;
11892        }
11893        tokio::time::sleep(poll).await;
11894    }
11895}
11896
11897/// Are any of `outcomes` [`CommandOutcome::resource_blocked`] - magi's own
11898/// admission that it could not even get a verify command to run, as opposed
11899/// to evidence the command actually produced? A caller that would otherwise
11900/// read a resource-blocked outcome as a red command must check this first:
11901/// see [`Runner::gate`], which retries rather than records `Blocked` when
11902/// this is true.
11903fn verify_inconclusive(outcomes: &[CommandOutcome]) -> bool {
11904    outcomes.iter().any(|o| o.resource_blocked)
11905}
11906
11907/// What [`Runner::gate_fix_round`] decided.
11908pub(crate) enum GateFix {
11909    /// The tree changed and `verify.e2e` is still green: run the gate again.
11910    Retry,
11911    /// No more rounds, nothing to fix, or the fix did not hold: the gate's
11912    /// last failure stands and the run ends blocked.
11913    Stop,
11914    /// `verify.e2e` could not run after the fix (magi's own contention):
11915    /// decide nothing now, a later reentry retries.
11916    Defer,
11917}
11918
11919/// Is every red command in `outcomes` an ordinary failure the code could
11920/// explain: it ran, exited non-zero, and said something?
11921///
11922/// A timeout, a spawn failure and a killed process all leave `code` `None`;
11923/// 126 / 127 are the POSIX shell's "cannot execute" / "not found". Output-free
11924/// exits carry nothing for a fixer to act on. Language-agnostic on purpose:
11925/// what the command is stays the gate's business.
11926pub(crate) fn gate_fixable(outcomes: &[CommandOutcome]) -> bool {
11927    let mut red = outcomes.iter().filter(|o| !o.ok()).peekable();
11928    red.peek().is_some()
11929        && red.all(|o| {
11930            !o.resource_blocked
11931                && matches!(o.code, Some(c) if c != 0 && c != 126 && c != 127)
11932                && !o.output_tail.trim().is_empty()
11933        })
11934}
11935
11936/// Describe one verify command's outcome for the event log, distinguishing a
11937/// build/link failure — the toolchain never produced a binary to run — from
11938/// an actual test failure, since only the latter is a verdict on the patch.
11939fn e2e_outcome_label(o: &CommandOutcome) -> String {
11940    if o.ok() {
11941        return "pass".to_owned();
11942    }
11943    let reason = if o.build_failed() {
11944        format!("COULD NOT RUN ({:?}, build/link failure)", o.code)
11945    } else {
11946        format!("FAIL ({:?})", o.code)
11947    };
11948    format!(
11949        "{reason}\n{}",
11950        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
11951    )
11952}
11953
11954/// Run `verify.e2e`, retrying once if the first attempt could not build or
11955/// link — a build/link failure is frequently a race against a shared
11956/// `CARGO_TARGET_DIR` (see AGENTS.md), not a verdict on the patch. Emits one
11957/// `verify` event per command, tagged with `context` (normally `"round N"`)
11958/// so the two call sites that need this — the ordinary per-round leg in
11959/// `review_loop`, and the deferred catch-up run `stop_reviewing` makes before
11960/// it will ever call a round green — read identically in the event log.
11961async fn run_e2e_with_retry(
11962    state: &mut RunState,
11963    shell: &[String],
11964    commands: &[String],
11965    worktree: &Path,
11966    timeout: Duration,
11967    context: &str,
11968) -> (Vec<CommandOutcome>, bool, Vec<u32>) {
11969    let (mut e2e, mut timed_out_pids) = run_commands(
11970        state, "verify", "e2e", 0, shell, commands, worktree, timeout,
11971    )
11972    .await;
11973    for o in &e2e {
11974        state.event(
11975            "verify",
11976            format!("{context}: `{}` -> {}", o.command, e2e_outcome_label(o)),
11977        );
11978    }
11979    // A build/link failure is not a verdict on the patch — it is frequently a
11980    // race against a shared `CARGO_TARGET_DIR` (see AGENTS.md). Give verify
11981    // one retry before letting a red like that decide the round.
11982    let verify_retried = e2e.iter().any(CommandOutcome::build_failed);
11983    if verify_retried {
11984        state.event(
11985            "verify",
11986            format!(
11987                "{context}: verify could not build/link, not a test result — retrying once \
11988                 before concluding"
11989            ),
11990        );
11991        let retried = run_commands(
11992            state, "verify", "e2e", 1, shell, commands, worktree, timeout,
11993        )
11994        .await;
11995        e2e = retried.0;
11996        // Both attempts' timeouts matter, not just the last one: the first
11997        // attempt's descendants may still be alive alongside the retry's.
11998        timed_out_pids.extend(retried.1);
11999        for o in &e2e {
12000            state.event(
12001                "verify",
12002                format!(
12003                    "{context}: retry `{}` -> {}",
12004                    o.command,
12005                    e2e_outcome_label(o)
12006                ),
12007            );
12008        }
12009    }
12010    (e2e, verify_retried, timed_out_pids)
12011}
12012
12013/// Run configured shell commands in `cwd`, in order. The second element is
12014/// the pid of every command that hit `timeout` and was still running when
12015/// this stopped waiting on it (best-effort: `None` when the platform did not
12016/// hand one back) — see [`with_cache_lease`]'s use of it for why a caller
12017/// that releases a shared resource afterward needs to know.
12018///
12019/// Records `task` into [`RunState::active`] at every command boundary
12020/// (`RunState::task_command`) and clears it once the whole list has run
12021/// (`RunState::task_finished`) — a `verify.e2e` / `verify.gate` list can run
12022/// for minutes with no seat and no output of its own to show for it (see
12023/// `CommandOutcome`'s doc on why an empty `e2e`/`gate` alone cannot be told
12024/// apart from "not yet run" without this), and this is the only place that
12025/// knows which command is running right now and how many are left. Three
12026/// saves per command — start, not per second — matching the same "only at a
12027/// boundary" rule [`wave`] already follows for seats.
12028#[allow(clippy::too_many_arguments)]
12029async fn run_commands(
12030    state: &mut RunState,
12031    node: &str,
12032    task: &str,
12033    attempt: usize,
12034    shell: &[String],
12035    commands: &[String],
12036    cwd: &Path,
12037    timeout: Duration,
12038) -> (Vec<CommandOutcome>, Vec<u32>) {
12039    if commands.is_empty() {
12040        // Nothing to mark as running and nothing to clear — an empty list
12041        // means "not configured", and touching `active` (or the disk) over
12042        // that would be a write for every round of a repo with no
12043        // `verify.e2e` / `verify.gate` commands at all.
12044        return (Vec::new(), Vec::new());
12045    }
12046    let mut out = Vec::new();
12047    let mut timed_out_pids = Vec::new();
12048    let total = commands.len();
12049    for (idx, command) in commands.iter().enumerate() {
12050        state.task_command(task, node, attempt, command, idx + 1, total, timeout);
12051        if let Err(e) = state.save() {
12052            tracing::warn!("could not persist an in-progress {task} command: {e:#}");
12053        }
12054        let started = Instant::now();
12055        let mut cmd = tokio::process::Command::new(&shell[0]);
12056        cmd.quiet();
12057        // Verify the checkout as a standalone caller. Worker ownership and
12058        // pinned executable selection belong to this driver, not to tests or
12059        // tools it runs. Preserve home/seat context and Running's containment.
12060        for key in crate::worker::INVOCATION_ENV {
12061            cmd.env_remove(key);
12062        }
12063        cmd.args(&shell[1..])
12064            .arg(command)
12065            .current_dir(cwd)
12066            .stdin(std::process::Stdio::null())
12067            .stdout(std::process::Stdio::piped())
12068            .stderr(std::process::Stdio::piped())
12069            .kill_on_drop(true);
12070        let mut outputs = Vec::new();
12071        let protected = (|| -> Result<()> {
12072            crate::incremental::configure(&mut cmd, cwd)?;
12073            let mut paths = vec![cwd.to_owned()];
12074            if let Some(cache) = state.config.cache_dir() {
12075                paths.push(cache);
12076            }
12077            outputs.extend(crate::retention::OutputLease::admit(
12078                &crate::run::home(),
12079                &paths,
12080                state.config.disk.min_free_bytes,
12081            )?);
12082            if let Some(guard) = crate::retention::OutputLease::borrow(cwd)? {
12083                outputs.push(guard);
12084            }
12085            if let Some(cache) = state.config.cache_dir() {
12086                outputs.push(crate::retention::OutputLease::cache(
12087                    &crate::run::home(),
12088                    &cache,
12089                    false,
12090                )?);
12091            }
12092            Ok(())
12093        })();
12094        if let Err(e) = protected {
12095            let detail = format!("execution admission unavailable: {e:#}");
12096            state.resource_block = Some(crate::run::ResourceBlock {
12097                cause: crate::run::ResourceBlockCause::Other,
12098                node: node.to_owned(),
12099                kind: if detail.contains("disk admission") {
12100                    "disk admission"
12101                } else {
12102                    "output ownership"
12103                }
12104                .to_owned(),
12105                detail: detail.clone(),
12106                at: Timestamp::now(),
12107            });
12108            out.push(CommandOutcome {
12109                command: command.clone(),
12110                code: None,
12111                output_tail: detail,
12112                duration_ms: 0,
12113                resource_blocked: true,
12114            });
12115            break;
12116        }
12117        let spawned = crate::seat_process::Running::spawn(cmd, None, outputs);
12118        let (code, body) = match spawned {
12119            Ok(mut running) => {
12120                let pid = running.child().id();
12121                let (stdout, out_reader) = agent::drain(running.child().stdout.take());
12122                let (stderr, err_reader) = agent::drain(running.child().stderr.take());
12123                let mut paths = vec![cwd.to_owned()];
12124                if let Some(cache) = state.config.cache_dir() {
12125                    paths.push(cache);
12126                }
12127                let mut pressure = None;
12128                let status = tokio::select! {
12129                    status = tokio::time::timeout(timeout, running.child().wait()) => Some(status),
12130                    reason = crate::retention::emergency(paths, state.config.disk.min_free_bytes) => {
12131                        pressure = Some(reason); None
12132                    }
12133                };
12134                let (code, failure) = match status {
12135                    Some(Ok(Ok(status))) => (
12136                        status.code(),
12137                        status
12138                            .code()
12139                            .is_none()
12140                            .then(|| format!("command terminated: {status}")),
12141                    ),
12142                    Some(Ok(Err(error))) => {
12143                        (None, Some(format!("failed waiting for command: {error}")))
12144                    }
12145                    Some(Err(_)) => {
12146                        // Leased cleanup confirms process-tree death itself.
12147                        // Unleased commands still need the caller's PID check
12148                        // after kill-on-drop, before releasing cache ownership.
12149                        if !running.coordinated()
12150                            && let Some(pid) = pid
12151                        {
12152                            timed_out_pids.push(pid);
12153                        }
12154                        (
12155                            None,
12156                            Some(format!("timed out after {}s", timeout.as_secs())),
12157                        )
12158                    }
12159                    None => (None, None),
12160                };
12161                running.finish().await;
12162                if let Some(reason) = pressure {
12163                    state.resource_block = Some(crate::run::ResourceBlock {
12164                        cause: crate::run::ResourceBlockCause::Other,
12165                        node: node.to_owned(),
12166                        kind: "disk pressure".to_owned(),
12167                        detail: reason.clone(),
12168                        at: Timestamp::now(),
12169                    });
12170                    out.push(CommandOutcome {
12171                        command: command.clone(),
12172                        code: None,
12173                        output_tail: reason,
12174                        duration_ms: started.elapsed().as_millis() as u64,
12175                        resource_blocked: true,
12176                    });
12177                    break;
12178                }
12179                let mut body = agent::collect(&stdout, out_reader, Duration::from_secs(2)).await;
12180                body.push_str(&agent::collect(&stderr, err_reader, Duration::from_secs(2)).await);
12181                if let Some(failure) = failure {
12182                    body.push('\n');
12183                    body.push_str(&failure);
12184                }
12185                (code, body)
12186            }
12187            Err(e) => (None, format!("failed to spawn `{}`: {e}", shell[0])),
12188        };
12189        let mut summary = summarize_verify(&body, crate::run::VERIFY_SUMMARY);
12190        if !body.is_empty() {
12191            // The full text is kept whatever the summary dropped. A fresh
12192            // name per command so a resume or retry never overwrites an
12193            // earlier run's output. A failed write changes no verdict.
12194            match crate::run::write_verification_artifact(state, task, &body) {
12195                Ok(name) if summary != body => {
12196                    summary = format!("[full output: artifacts/{name}]\n{summary}");
12197                }
12198                Ok(_) => {}
12199                Err(e) => {
12200                    let warning = format!("could not save the full {task} output: {e:#}");
12201                    tracing::warn!("{warning}");
12202                    state.event(node, warning);
12203                    if summary != body {
12204                        summary = format!("[full output could not be saved: {e:#}]\n{summary}");
12205                    }
12206                }
12207            }
12208        }
12209        let resource_blocked = code != Some(0) && body.contains(crate::incremental::BLOCKED);
12210        out.push(CommandOutcome {
12211            command: command.clone(),
12212            code,
12213            output_tail: summary,
12214            duration_ms: started.elapsed().as_millis() as u64,
12215            resource_blocked,
12216        });
12217        if resource_blocked {
12218            state.resource_block = Some(crate::run::ResourceBlock {
12219                cause: crate::run::ResourceBlockCause::Other,
12220                node: node.to_owned(),
12221                kind: "incremental cache admission".into(),
12222                detail: body,
12223                at: Timestamp::now(),
12224            });
12225            break;
12226        }
12227    }
12228    state.task_finished(task);
12229    if let Err(e) = state.save() {
12230        tracing::warn!("could not persist the end of {task}: {e:#}");
12231    }
12232    (out, timed_out_pids)
12233}
12234
12235/// The shell command line `mode = "none"` prints — in `magi show`'s `merge`
12236/// section (`report::run`) and in the `merge` event this node records — for
12237/// the operator to run by hand.
12238///
12239/// Built from [`MergeStyle`] rather than always `git merge --no-ff`: a base
12240/// branch whose ruleset forbids merge commits (GitHub's "must not contain
12241/// merge commits", or "require linear history") rejects the push a `--no-ff`
12242/// merge would produce, which is exactly the guidance this function replaces.
12243/// `message`'s first line becomes the squash commit's subject, matching the
12244/// note `report::run` prints alongside this command — see that function for
12245/// why an explicit subject is not optional there.
12246fn manual_merge_command(style: MergeStyle, repo: &Path, branch: &str, message: &str) -> String {
12247    let repo = repo.display();
12248    match style {
12249        MergeStyle::Merge => format!("git -C {repo} merge --no-ff {branch}"),
12250        MergeStyle::Squash => {
12251            // The subject sits inside double quotes, and a title an agent
12252            // wrote may carry the characters that break out of them.
12253            let subject = message
12254                .lines()
12255                .next()
12256                .unwrap_or(branch)
12257                .replace(['\\', '"', '$', '`'], "");
12258            format!(
12259                "git -C {repo} merge --squash {branch} && git -C {repo} commit -m \"{subject}\""
12260            )
12261        }
12262        MergeStyle::Rebase => format!("git -C {repo} merge --ff-only {branch}"),
12263    }
12264}
12265
12266/// GitHub's `createPullRequest` GraphQL mutation, which `gh pr create` calls
12267/// under the hood, rejects a `title` over 256 characters and the whole
12268/// command fails — no PR at all, for a run whose body was otherwise fine
12269/// (this is what happened to run 2963; see AGENTS.md). 240 leaves room below
12270/// that limit: titles are counted in `chars()` (Unicode scalars), which is not
12271/// always how GitHub counts. [`english_title`] keeps its trailing `...` inside
12272/// this bound. It is a margin, not a guarantee — a title packed
12273/// with multi-unit characters could still in principle land close to the
12274/// edge, but a real task title's occasional emoji or accented letter fits
12275/// comfortably inside it.
12276const PR_TITLE_MAX: usize = 240;
12277
12278/// A pull request title from the opening line of `text`, or `None` when that
12279/// line is not English (GitHub text is) or has no letters.
12280///
12281/// A line within `max` is kept as is. A longer one is cut at the end of its
12282/// first sentence when that falls inside `max`, else at a word boundary with a
12283/// plain `...` (ASCII, unlike the `…` `queue::title_from` appends, which would
12284/// make every merely-truncated title look non-English). The language check
12285/// runs on the kept text before any mark is added, so only what GitHub will
12286/// show is judged: an English opening followed by non-ASCII far past the cut
12287/// still passes.
12288fn english_title(text: &str, max: usize, ascii_only: bool) -> Option<String> {
12289    let line = queue::first_line(text)?;
12290    let chars: Vec<char> = line.chars().collect();
12291    let (kept, mark) = if chars.len() <= max {
12292        (line.to_owned(), "")
12293    } else if let Some(end) = sentence_end(&chars, max) {
12294        (chars[..end].iter().collect::<String>(), "")
12295    } else {
12296        let room = max.saturating_sub(3);
12297        // Cut at the last space inside the room; when the char just past the
12298        // room is a space the room already ends on a word.
12299        let cut = if chars[room].is_whitespace() {
12300            room
12301        } else {
12302            chars[..room]
12303                .iter()
12304                .rposition(|c| c.is_whitespace())
12305                .unwrap_or(room)
12306        };
12307        let head: String = chars[..cut].iter().collect();
12308        let head = head.trim_end_matches(|c: char| c.is_whitespace() || ",;:-".contains(c));
12309        (head.to_owned(), "...")
12310    };
12311    if kept.is_empty() {
12312        return None;
12313    }
12314    if ascii_only && (!kept.is_ascii() || !kept.chars().any(|c| c.is_ascii_alphabetic())) {
12315        return None;
12316    }
12317    if !ascii_only && !kept.chars().any(char::is_alphabetic) {
12318        return None;
12319    }
12320    Some(format!("{kept}{mark}"))
12321}
12322
12323/// The char length of the first sentence of `chars` when it ends within `max`
12324/// (the closing `.`/`!`/`?` dropped), skipping very short stubs and common
12325/// abbreviations so `e.g. foo` does not end a title early.
12326fn sentence_end(chars: &[char], max: usize) -> Option<usize> {
12327    const MIN: usize = 20;
12328    for i in MIN..max.min(chars.len()) {
12329        if !matches!(chars[i], '.' | '!' | '?') {
12330            continue;
12331        }
12332        let Some(&next) = chars.get(i + 1) else {
12333            continue;
12334        };
12335        if !next.is_whitespace() {
12336            continue;
12337        }
12338        let after = chars[i + 1..].iter().find(|c| !c.is_whitespace());
12339        if after.is_some_and(|c| c.is_ascii_lowercase()) {
12340            continue;
12341        }
12342        let word: String = chars[..i]
12343            .iter()
12344            .rev()
12345            .take_while(|c| !c.is_whitespace())
12346            .collect::<Vec<_>>()
12347            .into_iter()
12348            .rev()
12349            .collect();
12350        let word = word.to_ascii_lowercase();
12351        if matches!(word.as_str(), "e.g" | "i.e" | "etc" | "vs" | "cf") {
12352            continue;
12353        }
12354        let end = chars[..i]
12355            .iter()
12356            .rposition(|c| !c.is_whitespace())
12357            .map_or(i, |p| p + 1);
12358        return Some(end);
12359    }
12360    None
12361}
12362
12363/// What `merge = "pr"` (and the merge commit of the other modes) says about a
12364/// change: a title and a body describing what was *implemented*, not the task
12365/// that asked for it. A task reads as a request; a reader of the merged
12366/// history wants the change.
12367struct PrMessage {
12368    title: String,
12369    body: String,
12370}
12371
12372impl PrMessage {
12373    /// Title, blank line, body. The first line is the squash/merge commit
12374    /// subject (`manual_merge_command` takes it via `lines().next()`), so it
12375    /// has to stay one sensible line.
12376    fn commit_message(&self) -> String {
12377        format!("{}\n\n{}", self.title, self.body)
12378    }
12379}
12380
12381/// The text after a leading `TITLE:` (any case) on `line`.
12382fn title_marker(line: &str) -> Option<&str> {
12383    let line = line.trim();
12384    let head = line.get(..6)?;
12385    head.eq_ignore_ascii_case("title:")
12386        .then(|| line[6..].trim())
12387}
12388
12389/// The implementer's own one-line title: the `TITLE:` line the implement
12390/// prompt asks for at the top of its SUMMARY. Candidate commits are all
12391/// `magi: candidate X (uncommitted work)`, so a commit subject is never a
12392/// source, and a title that says as much is refused here too.
12393fn summary_title(summary: &str) -> Option<String> {
12394    let first = summary.lines().find(|l| !l.trim().is_empty())?;
12395    let raw = title_marker(first)?;
12396    if raw.is_empty() {
12397        return None;
12398    }
12399    let title = queue::title_from(raw, PR_TITLE_MAX);
12400    let lower = title.to_ascii_lowercase();
12401    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12402        return None;
12403    }
12404    Some(title)
12405}
12406
12407/// How `open_review`'s instruction begins; see [`landing_title`].
12408const REVIEW_PROMPT_OPENING: &str = "Review the work already on branch";
12409
12410/// Marker `open_review` gives a candidate that nothing in the roster wrote.
12411pub(crate) const EXISTING_BRANCH: &str = "(existing branch)";
12412
12413/// Does this run review work that already existed, rather than implement a
12414/// task? Runs recorded before `reviewed_commits` existed carry only the
12415/// candidate marker.
12416pub(crate) fn is_review_run(state: &RunState) -> bool {
12417    state.reviewed_commits.is_some() || state.candidates.iter().any(|c| c.agent == EXISTING_BRANCH)
12418}
12419
12420/// The title of a review-only run: the subject of the oldest commit under
12421/// review. Later commits are usually fixups, and `instruction` is the review
12422/// prompt, which says nothing about the change. GitHub text is English, so a
12423/// non-ASCII or blank subject yields `None` and the caller's neutral title.
12424fn review_title(state: &RunState) -> Option<String> {
12425    english_subject(
12426        state.reviewed_commits.as_ref()?.first()?,
12427        state.config.graph.github_english(),
12428    )
12429}
12430
12431/// `raw` as a pull request title, or `None` when it is blank, not English
12432/// (GitHub text is), or one of magi's own candidate commit subjects.
12433fn english_subject(raw: &str, ascii_only: bool) -> Option<String> {
12434    let raw = raw.trim();
12435    if raw.is_empty() {
12436        return None;
12437    }
12438    if ascii_only && (!raw.is_ascii() || !raw.chars().any(|c| c.is_ascii_alphabetic())) {
12439        return None;
12440    }
12441    if !ascii_only && !raw.chars().any(char::is_alphabetic) {
12442        return None;
12443    }
12444    let title = queue::title_from(raw, PR_TITLE_MAX);
12445    let lower = title.to_ascii_lowercase();
12446    if lower.starts_with("magi:") || lower.contains("(uncommitted work)") {
12447        return None;
12448    }
12449    Some(title)
12450}
12451
12452/// What a review-only run's branch says about itself, read at the moment the
12453/// pull request is opened.
12454#[derive(Debug, Clone, PartialEq, Eq)]
12455struct BranchFacts {
12456    /// `(subject, body)` of each commit, oldest first.
12457    commits: Vec<(String, String)>,
12458    /// Trimmed `git diff --stat`.
12459    stat: String,
12460}
12461
12462/// Longest diff stat shown: this many file lines plus the summary line.
12463const STAT_FILE_LINES: usize = 25;
12464/// Cap on the commit list, well inside GitHub's 65536-character body limit.
12465const COMMITS_MAX_CHARS: usize = 20_000;
12466
12467/// Read the commits and diff stat of `base..branch`. `None` when git cannot
12468/// say or finds nothing, so the caller falls back to what the run recorded.
12469async fn branch_facts(repo: &Path, base: &str, branch: &str) -> Option<BranchFacts> {
12470    let commits = git::commit_log(repo, base, branch).await.ok()?;
12471    if commits.is_empty() {
12472        return None;
12473    }
12474    let stat = git::diff_stat(repo, base, branch).await.unwrap_or_default();
12475    let lines: Vec<&str> = stat.lines().collect();
12476    let stat = if lines.len() > STAT_FILE_LINES + 1 {
12477        let omitted = lines.len() - 1 - STAT_FILE_LINES;
12478        let more = format!(" ... {omitted} more file(s)");
12479        let mut kept: Vec<&str> = lines[..STAT_FILE_LINES].to_vec();
12480        kept.push(&more);
12481        kept.push(lines[lines.len() - 1]);
12482        kept.join("\n")
12483    } else {
12484        lines.join("\n")
12485    };
12486    Some(BranchFacts { commits, stat })
12487}
12488
12489/// Defang what would break the surrounding markdown: a closing `</details>`
12490/// and a code fence.
12491fn markdown_safe(text: &str) -> String {
12492    text.replace("</details>", "&lt;/details&gt;")
12493        .replace("\x60\x60\x60", "~~~")
12494}
12495
12496fn neutral_title(state: &RunState, winner: char) -> String {
12497    format!(
12498        "chore: land candidate {} of run {}",
12499        winner.to_ascii_uppercase(),
12500        state.id
12501    )
12502}
12503
12504/// Reuse the graph's bounded fixer without starting another graph or driver.
12505/// Copy the working state back even on errors so spent rounds remain visible.
12506pub(crate) async fn repair_landing_gate(
12507    state: &mut RunState,
12508    winner: &Candidate,
12509    outcomes: &[CommandOutcome],
12510) -> Result<GateFix> {
12511    if state.gate_fixes.len() >= state.config.graph.gate_fix_rounds || !gate_fixable(outcomes) {
12512        return Ok(GateFix::Stop);
12513    }
12514    let mut runner = Runner {
12515        roles: state.config.resolve_roles()?,
12516        sem: Arc::new(Semaphore::new(state.config.graph.max_parallel.max(1))),
12517        state: state.clone(),
12518        pause: Pause::new(),
12519        interrupt: Pause::new(),
12520    };
12521    let result = runner.gate_fix_round(winner, outcomes).await;
12522    *state = runner.state;
12523    result
12524}
12525
12526/// Review judgment survives a clean rebase; command evidence belongs to a SHA.
12527/// A completed failure is terminal on that SHA. Resource waits remain retryable.
12528pub(crate) async fn verify_current_head(
12529    state: &mut RunState,
12530    winner: &Candidate,
12531    wait: &crate::repo_lock::WaitPolicy,
12532) -> Result<bool> {
12533    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12534    // Preserve the previous record before replacing it, including on crash recovery.
12535    if state.head_e2e.as_ref().is_some_and(|e| e.head != head) {
12536        state
12537            .verification_history
12538            .push(crate::run::VerificationEvidence {
12539                superseded_by: Some(head.clone()),
12540                archived_at: Some(Timestamp::now()),
12541                fixes_spent: Some(state.gate_fixes.len()),
12542                gate_head: None,
12543                gate_ran: false,
12544                gate: Vec::new(),
12545                e2e: state.head_e2e.take(),
12546            });
12547    }
12548    if let Some(e) = &state.head_e2e
12549        && e.head == head
12550    {
12551        if e.passed {
12552            return Ok(true);
12553        }
12554        if !e.outcomes.is_empty() && !verify_inconclusive(&e.outcomes) {
12555            state.status = RunStatus::Blocked;
12556            state.save()?;
12557            return Ok(false);
12558        }
12559    } else if state.reviews.last().is_some_and(|r| {
12560        r.verified_head.as_deref() == Some(head.as_str())
12561            && (r.e2e_status() == E2eStatus::Passed
12562                || (r.e2e_status() == E2eStatus::NotConfigured
12563                    && state.config.verify.e2e.is_empty()))
12564    }) || state
12565        .gate_repair_e2e
12566        .as_ref()
12567        .is_some_and(|e| e.head == head && e.passed)
12568    {
12569        return Ok(true);
12570    }
12571    if let Some(e) = &state.gate_repair_e2e
12572        && e.head == head
12573        && !e.passed
12574        && !e.outcomes.is_empty()
12575        && !verify_inconclusive(&e.outcomes)
12576    {
12577        state.status = RunStatus::Blocked;
12578        state.save()?;
12579        return Ok(false);
12580    }
12581    state.head_e2e = Some(crate::run::GateRepairE2e {
12582        head: head.clone(),
12583        passed: false,
12584        outcomes: Vec::new(),
12585    });
12586    state.save()?;
12587    let commands = state.config.verify.e2e.clone();
12588    let shell = state.config.shell();
12589    let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12590    let cache_dir = state.config.cache_dir();
12591    let outcomes = if commands.is_empty() {
12592        Vec::new()
12593    } else {
12594        let (outcomes, _) = with_cache_lease(
12595            state,
12596            cache_dir.as_deref(),
12597            "e2e",
12598            "e2e",
12599            &winner.worktree,
12600            &head,
12601            timeout,
12602            "current head e2e",
12603            wait,
12604            |state, budget| async move {
12605                run_e2e_with_retry(
12606                    state,
12607                    &shell,
12608                    &commands,
12609                    &winner.worktree,
12610                    budget,
12611                    "current head e2e",
12612                )
12613                .await
12614            },
12615        )
12616        .await?;
12617        outcomes
12618    };
12619    let unresolved = verify_inconclusive(&outcomes);
12620    let passed = !unresolved
12621        && outcomes.iter().all(CommandOutcome::ok)
12622        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12623    if !outcomes.is_empty() && !unresolved {
12624        state.record_base_progress(&head);
12625    }
12626    state.head_e2e = Some(crate::run::GateRepairE2e {
12627        head: head.clone(),
12628        passed,
12629        outcomes,
12630    });
12631    if !passed && !unresolved {
12632        state.status = RunStatus::Blocked;
12633        state.event(
12634            "e2e",
12635            format!("verify.e2e failed on head {head}; nothing published"),
12636        );
12637    }
12638    state.save()?;
12639    Ok(passed)
12640}
12641
12642/// Recheck a repair's e2e requirement across failures, resource deferrals and
12643/// process interruptions. A successful result only applies to its exact SHA.
12644pub(crate) async fn check_gate_repair_e2e(
12645    state: &mut RunState,
12646    winner: &Candidate,
12647    wait: &crate::repo_lock::WaitPolicy,
12648) -> Result<GateFix> {
12649    if state.gate_repair_e2e.is_none() && state.gate_fixes.is_empty() {
12650        return Ok(GateFix::Retry);
12651    }
12652    let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12653    if state
12654        .gate_repair_e2e
12655        .as_ref()
12656        .is_some_and(|r| r.head == head && r.passed)
12657    {
12658        return Ok(GateFix::Retry);
12659    }
12660    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12661        head: head.clone(),
12662        passed: false,
12663        outcomes: Vec::new(),
12664    });
12665    state.save()?;
12666    let commands = state.config.verify.e2e.clone();
12667    let outcomes = if commands.is_empty() {
12668        Vec::new()
12669    } else {
12670        let shell = state.config.shell();
12671        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12672        let cache_dir = state.config.cache_dir();
12673        let (outcomes, _) = with_cache_lease(
12674            state,
12675            cache_dir.as_deref(),
12676            "e2e",
12677            "e2e",
12678            &winner.worktree,
12679            &head,
12680            timeout,
12681            "gate repair e2e",
12682            wait,
12683            |state, budget| async move {
12684                run_e2e_with_retry(
12685                    state,
12686                    &shell,
12687                    &commands,
12688                    &winner.worktree,
12689                    budget,
12690                    "gate repair e2e",
12691                )
12692                .await
12693            },
12694        )
12695        .await?;
12696        outcomes
12697    };
12698    let deferred = verify_inconclusive(&outcomes);
12699    let passed = outcomes.iter().all(CommandOutcome::ok)
12700        && git::rev_parse(&winner.worktree, "HEAD").await? == head;
12701    state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
12702        head,
12703        passed,
12704        outcomes,
12705    });
12706    state.save()?;
12707    if deferred {
12708        Ok(GateFix::Defer)
12709    } else if !passed {
12710        state.event(
12711            "gate",
12712            "verify.e2e failed after gate repair; nothing published",
12713        );
12714        Ok(GateFix::Stop)
12715    } else {
12716        Ok(GateFix::Retry)
12717    }
12718}
12719
12720/// Run the same configured gate for a landing head and a fresh graph.
12721pub(crate) async fn run_landing_gate(
12722    state: &mut RunState,
12723    winner: &Candidate,
12724    wait: &crate::repo_lock::WaitPolicy,
12725) -> Result<Vec<CommandOutcome>> {
12726    let shell = state.config.shell();
12727    let gate_commands = state.config.verify.gate.clone();
12728    let has_pre_gate = !state.config.verify.pre_gate.is_empty();
12729    let outcomes = if gate_commands.is_empty() && !has_pre_gate {
12730        state.gate_head = Some(git::rev_parse(&winner.worktree, "HEAD").await?);
12731        Vec::new()
12732    } else {
12733        let timeout = Duration::from_secs(state.config.graph.verify_timeout());
12734        let cache_dir = state.config.cache_dir();
12735        let head = git::rev_parse(&winner.worktree, "HEAD").await?;
12736        let (outcomes, _) = with_cache_lease(
12737            state,
12738            cache_dir.as_deref(),
12739            "gate",
12740            "gate",
12741            &winner.worktree,
12742            &head,
12743            timeout,
12744            "final gate",
12745            wait,
12746            |state, budget| {
12747                let shell = shell.clone();
12748                let gate_commands = gate_commands.clone();
12749                let winner = winner.clone();
12750                async move {
12751                    Runner::run_pre_gate(state, &winner).await;
12752                    // Bind command evidence after mechanical fixes, before commands start.
12753                    state.gate_head = git::rev_parse(&winner.worktree, "HEAD").await.ok();
12754                    let (outcomes, timed_out_pids) = run_commands(
12755                        state,
12756                        "gate",
12757                        "gate",
12758                        0,
12759                        &shell,
12760                        &gate_commands,
12761                        &winner.worktree,
12762                        budget,
12763                    )
12764                    .await;
12765                    (outcomes, false, timed_out_pids)
12766                }
12767            },
12768        )
12769        .await?;
12770        outcomes
12771    };
12772    let current = git::rev_parse(&winner.worktree, "HEAD").await?;
12773    if !verify_inconclusive(&outcomes) && state.gate_head.as_deref() != Some(current.as_str()) {
12774        let mut outcomes = outcomes;
12775        outcomes.push(CommandOutcome {
12776            command: "gate head stability".into(), code: None,
12777            output_tail: "winner changed during gate, or checked head is unknown; fresh verification required".into(),
12778            duration_ms: 0, resource_blocked: false,
12779        });
12780        return Ok(outcomes);
12781    }
12782    if outcomes.is_empty() {
12783        // Nothing configured to check — distinct from every other
12784        // silence in this run's event log, since an empty `gate` alone
12785        // no longer says whether the gate ran at all (see
12786        // `RunState::gate_ran`'s own doc).
12787        state.event("gate", "no gate commands configured; not tested");
12788    }
12789    for o in &outcomes {
12790        state.event(
12791            "gate",
12792            format!(
12793                "`{}` -> {}",
12794                o.command,
12795                if o.ok() {
12796                    "pass".to_owned()
12797                } else {
12798                    format!(
12799                        "FAIL ({:?})\n{}",
12800                        o.code,
12801                        summarize_verify(&o.output_tail, EVENT_OUTPUT_TAIL)
12802                    )
12803                }
12804            ),
12805        );
12806    }
12807    Ok(outcomes)
12808}
12809
12810/// The pull request title to hand to `land::merge_subject`. A review-only run
12811/// opened by an earlier build titled its pull request with the review prompt;
12812/// that title is dropped (empty, so the fallback applies) rather than landed.
12813/// Any other title, including an operator's rename, passes through untouched,
12814/// and so does every title of a run that implements a task.
12815pub fn landing_title<'a>(state: &RunState, pr_title: &'a str) -> &'a str {
12816    if is_review_run(state) && pr_title.trim_start().starts_with(REVIEW_PROMPT_OPENING) {
12817        ""
12818    } else {
12819        pr_title
12820    }
12821}
12822
12823/// What the squash subject falls back to when the pull request title is empty
12824/// or candidate-shaped: for a review-only run the derived title, never the
12825/// review prompt held in `instruction`.
12826pub fn landing_subject_source(state: &RunState) -> String {
12827    if is_review_run(state) {
12828        let winner = state.candidates.first().map_or('A', |c| c.label);
12829        return review_title(state).unwrap_or_else(|| neutral_title(state, winner));
12830    }
12831    state.instruction.clone()
12832}
12833
12834/// `summary` without its `TITLE:` line, which the pull request title already
12835/// carries.
12836fn summary_without_title(summary: &str) -> String {
12837    let mut lines = summary.trim().lines().peekable();
12838    if lines.peek().is_some_and(|l| title_marker(l).is_some()) {
12839        lines.next();
12840    }
12841    lines.collect::<Vec<_>>().join("\n").trim().to_owned()
12842}
12843
12844/// The pull request title and body for the winning candidate.
12845///
12846/// Title: the implementer's `TITLE:` line ([`summary_title`]), falling back to
12847/// the task's own opening line via [`queue::title_from`] when there is none.
12848/// `state.instruction` can open with blank lines (`task_text` only rejects a
12849/// body that is blank *entirely*), which `title_from` skips.
12850///
12851/// Publishing uses an independent description generated from the final diff
12852/// and structured evidence. Internal summaries and reports remain local.
12853/// Non-publishing previews may display the original task and candidate summary.
12854/// The footer retains the run and candidate provenance.
12855#[cfg(test)]
12856fn pr_message(state: &RunState, winner: char) -> PrMessage {
12857    pr_message_with(state, winner, None)
12858}
12859
12860/// [`pr_message`] with what the branch of a review-only run says about itself.
12861/// `facts` is ignored for a run that implements a task.
12862#[cfg(test)]
12863fn pr_message_with(state: &RunState, winner: char, facts: Option<&BranchFacts>) -> PrMessage {
12864    let raw = pr_message_raw(
12865        state,
12866        winner,
12867        facts,
12868        crate::pr_outcome::FinalHead::Unreported,
12869        ChangeNote::NotAsked,
12870    );
12871    let id = crate::scrub::Identity::current();
12872    PrMessage {
12873        title: crate::scrub::scrub(&raw.title, &id),
12874        body: crate::scrub::scrub(&raw.body, &id),
12875    }
12876}
12877
12878/// What the public description may say about the change itself.
12879#[derive(Debug, Clone, Copy)]
12880enum ChangeNote<'a> {
12881    /// A local preview or non-publishing merge mode: no public generation needed.
12882    NotAsked,
12883    /// A summary written from the final diff.
12884    Summary(&'a crate::run::ChangeSummary),
12885    /// A summary was asked for and could not be produced; the text says why.
12886    Unavailable(&'a str),
12887}
12888
12889fn pr_message_raw(
12890    state: &RunState,
12891    winner: char,
12892    facts: Option<&BranchFacts>,
12893    head: crate::pr_outcome::FinalHead<'_>,
12894    change: ChangeNote<'_>,
12895) -> PrMessage {
12896    if !matches!(change, ChangeNote::NotAsked) {
12897        let (title, mut body) = match change {
12898            ChangeNote::Summary(c) => (
12899                c.title.clone(),
12900                format!("## Summary\n\n{}\n\n", c.body.trim()),
12901            ),
12902            ChangeNote::Unavailable(why) => (
12903                neutral_title(state, winner),
12904                format!(
12905                    "## Summary\n\nNo description of what this change does could be generated ({why}). Review the final diff; internal reports remain in local records.\n\n"
12906                ),
12907            ),
12908            ChangeNote::NotAsked => unreachable!(),
12909        };
12910        body.push_str(&public_pr_evidence(state, head));
12911        body.push_str(&format!(
12912            "\n---\nmagi:run/{} magi:candidate-{}\n",
12913            state.id,
12914            winner.to_ascii_lowercase()
12915        ));
12916        return PrMessage { title, body };
12917    }
12918    pr_preview(state, winner, facts, head)
12919}
12920
12921/// Append public facts from the run records after every prose generation,
12922/// including posting-gate rewrites. The agent cannot drop this provenance.
12923fn public_pr_evidence(state: &RunState, head: crate::pr_outcome::FinalHead<'_>) -> String {
12924    let mut body = crate::pr_outcome::review_rounds(state);
12925    body.push_str(&crate::pr_outcome::final_verification(state, head));
12926    let open = state.last_round_findings();
12927    if !open.is_empty() {
12928        body.push_str("## Open review findings\n\n");
12929        for f in open {
12930            body.push_str(&format!("- `{}` [{:?}]\n", f.id, f.severity));
12931        }
12932        body.push('\n');
12933    }
12934    if let Some(c) = state.change_summary.as_ref()
12935        && head == crate::pr_outcome::FinalHead::At(&c.head)
12936    {
12937        body.push_str(&format!(
12938            "Evidence: final diff `{}`..`{}`; review and verification records from this run.\n",
12939            c.base, c.head
12940        ));
12941    }
12942    body
12943}
12944
12945/// Local-only preview of the operator's task and candidate summary.
12946fn pr_preview(
12947    state: &RunState,
12948    winner: char,
12949    facts: Option<&BranchFacts>,
12950    head: crate::pr_outcome::FinalHead<'_>,
12951) -> PrMessage {
12952    let summary = state
12953        .candidates
12954        .iter()
12955        .find(|c| c.label == winner)
12956        .map(|c| c.summary.as_str())
12957        .unwrap_or_default();
12958    // The fallback is the operator's own words and may not be English; GitHub
12959    // text always is, so a non-English task gets a neutral title instead.
12960    let review = is_review_run(state);
12961    let title = if review {
12962        facts
12963            .and_then(|f| {
12964                english_subject(&f.commits.first()?.0, state.config.graph.github_english())
12965            })
12966            .or_else(|| review_title(state))
12967            .or_else(|| {
12968                state
12969                    .candidates
12970                    .iter()
12971                    .find(|c| c.label == winner)
12972                    .filter(|c| !c.branch.starts_with("magi/"))
12973                    .and_then(|c| english_subject(&c.branch, state.config.graph.github_english()))
12974            })
12975            .unwrap_or_else(|| neutral_title(state, winner))
12976    } else {
12977        summary_title(summary).unwrap_or_else(|| {
12978            english_title(
12979                &state.instruction,
12980                PR_TITLE_MAX,
12981                state.config.graph.github_english(),
12982            )
12983            .unwrap_or_else(|| neutral_title(state, winner))
12984        })
12985    };
12986
12987    let mut body = String::new();
12988    let what = summary_without_title(summary);
12989    if !what.is_empty() {
12990        body.push_str("## Summary\n\n");
12991        body.push_str(&what);
12992        body.push_str("\n\n");
12993    }
12994
12995    // Review history uses structured facts across all rounds, never reports.
12996    body.push_str(&crate::pr_outcome::review_rounds(state));
12997    body.push_str(&crate::pr_outcome::final_verification(state, head));
12998    let fix = state.reviews.iter().rev().find_map(|r| r.fix.as_ref());
12999
13000    let open = state.open_findings();
13001    if !open.is_empty() {
13002        body.push_str("## Open review findings\n\n");
13003        for f in &open {
13004            body.push_str(&format!(
13005                "- `{}` [{:?}] {}\n",
13006                f.id, f.severity, "Context retained in local review records."
13007            ));
13008        }
13009        body.push('\n');
13010    }
13011
13012    if let Some(fix) = fix
13013        && !fix.rejected.is_empty()
13014    {
13015        body.push_str("## Declined by the fixer\n\n");
13016        for r in &fix.rejected {
13017            body.push_str(&format!(
13018                "- `{}`: {}\n",
13019                r.id, "Reason retained in local review records."
13020            ));
13021        }
13022        body.push('\n');
13023    }
13024
13025    if review {
13026        // The review prompt is not the task; list what the branch carries.
13027        body.push_str("## Commits under review\n\n");
13028        if let Some(facts) = facts {
13029            let mut left = COMMITS_MAX_CHARS;
13030            for (i, (subject, text)) in facts.commits.iter().enumerate() {
13031                let mut entry = format!("- {}\n", markdown_safe(subject));
13032                for l in markdown_safe(text).lines() {
13033                    entry.push_str(format!("  {l}\n").trim_end_matches(' '));
13034                }
13035                if left == 0 {
13036                    body.push_str(&format!(
13037                        "- ... {} more commit(s)\n",
13038                        facts.commits.len() - i
13039                    ));
13040                    break;
13041                }
13042                if entry.len() > left {
13043                    // Even the first commit is cut: one huge body must not
13044                    // push the whole description past GitHub's limit.
13045                    let mut end = left;
13046                    while !entry.is_char_boundary(end) {
13047                        end -= 1;
13048                    }
13049                    entry.truncate(end);
13050                    entry.push_str("\n  ... (truncated)\n");
13051                    left = 0;
13052                } else {
13053                    left -= entry.len();
13054                }
13055                body.push_str(&entry);
13056            }
13057            if !facts.stat.trim().is_empty() {
13058                body.push_str(&format!(
13059                    "\n## Diff stat\n\n```\n{}\n```\n",
13060                    markdown_safe(facts.stat.trim())
13061                ));
13062            }
13063        } else {
13064            match &state.reviewed_commits {
13065                Some(subjects) => {
13066                    for s in subjects {
13067                        body.push_str(&format!("- {}\n", s.trim()));
13068                    }
13069                }
13070                None => {
13071                    // An older run kept only the prompt, with the commit list
13072                    // after its first paragraph.
13073                    let rest = state.instruction.split_once("\n\n").map_or("", |(_, r)| r);
13074                    body.push_str(rest.trim());
13075                    body.push('\n');
13076                }
13077            }
13078        }
13079    } else {
13080        let task = state.instruction.trim();
13081        let task = if task.is_empty() {
13082            "(empty task)"
13083        } else {
13084            task
13085        };
13086        body.push_str(&format!(
13087            "<details>\n<summary>Original task</summary>\n\n{}\n\n</details>\n",
13088            task.replace("</details>", "&lt;/details&gt;")
13089        ));
13090    }
13091
13092    body.push_str(&format!(
13093        "\n---\nmagi:run/{} magi:candidate-{}\n",
13094        state.id,
13095        winner.to_ascii_lowercase()
13096    ));
13097
13098    PrMessage { title, body }
13099}
13100
13101/// Source imports have already been prepared from structured inputs. Legacy
13102/// inferred references must never instruct a seat to reconstruct those imports.
13103fn seeded_instruction(state: &RunState) -> String {
13104    let mut inputs = Vec::new();
13105    if let Some(start) = &state.sources.start_from {
13106        inputs.push(format!(
13107            "Start from `{}` at `{}` (including ancestry).",
13108            start.token, start.sha
13109        ));
13110    }
13111    for pick in &state.sources.pick_commits {
13112        inputs.push(format!(
13113            "Pick `{}` at `{}` (single commit).",
13114            pick.token, pick.sha
13115        ));
13116    }
13117    if inputs.is_empty() {
13118        state.instruction.clone()
13119    } else {
13120        format!(
13121            "{}\n\n# Explicit source inputs already prepared\n\n{}",
13122            state.instruction,
13123            inputs.join("\n")
13124        )
13125    }
13126}
13127
13128/// Does the winner have no commits ahead of the base it would land on?
13129/// Any failure to find out reads as "not empty": the merge then behaves as it
13130/// always did rather than refusing on a guess.
13131async fn merge_is_empty(repo: &Path, state: &RunState, branch: &str, mode: MergeMode) -> bool {
13132    let base = &state.base_branch;
13133    let mut against = base.clone();
13134    if mode == MergeMode::Pr {
13135        // A pull request lands on the remote's base, never the local branch
13136        // of the same name: if that cannot be read, "not empty" is the safe
13137        // answer.
13138        let remote = &state.config.merge.remote;
13139        let tracking = format!("{remote}/{base}");
13140        let fetched = git::fetch(repo, remote, base).await;
13141        if fetched.is_ok_and(|o| o.ok()) && git::rev_exists(repo, &tracking).await {
13142            against = tracking;
13143        } else {
13144            return false;
13145        }
13146    }
13147    matches!(git::commits_ahead(repo, &against, branch).await, Ok(0))
13148}
13149
13150/// Why nothing was opened for an empty winner, with what the task's own
13151/// references resolved to.
13152fn empty_candidate_detail(_state: &RunState, base: &str) -> String {
13153    let mut detail = format!(
13154        "empty candidate: the winning branch has 0 commits ahead of {base}, so there is \
13155         nothing to open a pull request for"
13156    );
13157    detail.push_str(
13158        "\nSource imports require explicit --start-from or --pick-commit options on a new task.",
13159    );
13160    detail
13161}
13162
13163/// What the `Pr` merge does once it knows whether the branch already has an
13164/// open pull request.
13165#[derive(Debug, PartialEq, Eq)]
13166enum PrPlan {
13167    Create,
13168    Adopt { url: String, title: String },
13169    Stop(String),
13170}
13171
13172/// Pure decision behind the `Pr` merge: none -> create, one -> adopt, many or
13173/// a failed lookup -> stop with the real reason. Never guesses.
13174fn pr_merge_plan(found: Result<land::OpenPr>) -> PrPlan {
13175    match found {
13176        Ok(land::OpenPr::None) => PrPlan::Create,
13177        Ok(land::OpenPr::One { url, title }) => PrPlan::Adopt { url, title },
13178        Ok(land::OpenPr::Many(urls)) => PrPlan::Stop(format!(
13179            "several open pull requests exist for this branch, not picking one: {}",
13180            urls.join(" ")
13181        )),
13182        Err(e) => PrPlan::Stop(format!("could not look up open pull requests: {e:#}")),
13183    }
13184}
13185
13186/// `gh pr create`, returning the PR url.
13187pub(crate) async fn gh_pr_create(
13188    cwd: &Path,
13189    base: &str,
13190    head: &str,
13191    title: &str,
13192    body: &str,
13193) -> Result<String> {
13194    let out = tokio::process::Command::new("gh")
13195        .args([
13196            "pr", "create", "--base", base, "--head", head, "--title", title, "--body", body,
13197        ])
13198        .current_dir(cwd)
13199        .quiet()
13200        .stdin(std::process::Stdio::null())
13201        .output()
13202        .await
13203        .context("spawn gh")?;
13204    if out.status.success() {
13205        Ok(String::from_utf8_lossy(&out.stdout).trim().to_owned())
13206    } else {
13207        bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned())
13208    }
13209}
13210
13211/// Never force deletion of uncommitted work, including unknown legacy trees.
13212async fn remove_preserved_worktree(repo: &Path, path: &Path) -> Result<()> {
13213    match std::fs::symlink_metadata(path) {
13214        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
13215            // A previous cleanup may have removed the tree before a branch
13216            // deletion or state save failed. Unregister it if still present.
13217            git::worktree_remove_clean(repo, path).await?;
13218            return Ok(());
13219        }
13220        Err(e) => return Err(e.into()),
13221        Ok(_) => {}
13222    }
13223    if !git::is_clean(path).await? {
13224        bail!(
13225            "worktree has uncommitted work; refusing removal: {}",
13226            path.display()
13227        );
13228    }
13229    if !git::worktree_remove_clean(repo, path).await? {
13230        bail!("worktree removal refused: {}", path.display());
13231    }
13232    Ok(())
13233}
13234
13235/// Tear a run's worktrees and branches down.
13236///
13237/// `home` is where the updated `run.json` is saved (via
13238/// [`RunState::save_under`]), never the process-global [`crate::run::home`]:
13239/// a housekeeping pass already has its own honest `home` handed to it, and
13240/// falling through to the global here would write back through whichever
13241/// directory some other process or test pinned into that `OnceLock` first,
13242/// not the one the caller actually resolved its `runs` and `state` from.
13243pub async fn fold_run(state: &mut RunState, drop_winner: bool, home: &Path) -> Result<Vec<String>> {
13244    if let Some(reason) = crate::pr_recovery::source_retention_reason(home, state) {
13245        bail!("refusing fold: {reason}");
13246    }
13247    if !state.candidate_preservation.is_empty() || !state.interrupted_implementations.is_empty() {
13248        bail!("candidate preservation incomplete; refusing cleanup");
13249    }
13250    // Serialize ordinary folding with the same starts/resumes as target-only
13251    // eviction. Otherwise a second janitor could remove the whole worktree
13252    // around a live output borrower even though target eviction was fenced.
13253    std::fs::create_dir_all(home.join("runs").join(&state.id))?;
13254    let _owner = crate::pr_head::owner_lock(home, &state.id)?;
13255    let _driver = crate::driver::maintenance(home, &state.id)?;
13256    let mut output_guards = Vec::new();
13257    for entry in std::fs::read_dir(state.worktree_root())
13258        .into_iter()
13259        .flatten()
13260        .flatten()
13261    {
13262        if entry.path().join(".git").is_file() {
13263            output_guards.push(crate::retention::OutputLease::take(&entry.path(), true)?);
13264        }
13265    }
13266    let repo = state.repo.clone();
13267    // CLI, Web and automatic cleanup all enter here, independently of
13268    // Runner::fold_losers. Protect deletion and shared config release too.
13269    let wait = repository_wait(state, None);
13270    let _coordination = match coordinate_operation(state, &repo, &wait).await {
13271        Ok(lease) => lease,
13272        Err(e) => {
13273            state.save_under(home)?;
13274            return Err(e);
13275        }
13276    };
13277    // Confirmed merges dispose of their owned sources after durable evidence
13278    // capture. Preflight every source before removing anything.
13279    let _source_outputs = if state.status == RunStatus::Merged
13280        && state.merge.as_ref().is_some_and(|merge| merge.ok)
13281    {
13282        verify_merged_fold_sources(state, drop_winner, home).await?
13283    } else {
13284        Vec::new()
13285    };
13286    let root = state.worktree_root();
13287    let winner = state.tally.as_ref().map(|t| t.winner);
13288    let mut removed = Vec::new();
13289
13290    for i in 0..state.candidates.len() {
13291        let c = state.candidates[i].clone();
13292        let is_winner = Some(c.label) == winner;
13293        if is_winner && !drop_winner {
13294            continue;
13295        }
13296        let existed = c.worktree.exists();
13297        // A vanished directory can still have a Git registration that keeps
13298        // its branch checked out. Remove that registration before the branch.
13299        remove_preserved_worktree(&repo, &c.worktree).await?;
13300        if existed {
13301            removed.push(c.worktree.to_string_lossy().into_owned());
13302        }
13303        // A branch handed to a later run (and its pull request) is not this
13304        // run's to delete.
13305        let handed_over = state.released_branches.contains(&c.branch);
13306        if !handed_over && git::branch_exists(&repo, &c.branch).await? {
13307            if !git::branch_delete(&repo, &c.branch).await? {
13308                bail!("candidate branch removal refused");
13309            }
13310            removed.push(c.branch.clone());
13311        }
13312        state.candidates[i].folded = true;
13313    }
13314
13315    for name in std::fs::read_dir(&root).into_iter().flatten().flatten() {
13316        let path = name.path();
13317        let keep = !drop_winner
13318            && winner.is_some_and(|w| {
13319                path.file_name()
13320                    .is_some_and(|n| n == format!("cand-{w}").as_str())
13321            });
13322        if keep {
13323            continue;
13324        }
13325        remove_preserved_worktree(&repo, &path).await?;
13326        removed.push(path.to_string_lossy().into_owned());
13327    }
13328
13329    // `root` (`wt/<...>/<short>/`) held nothing but this run's candidate and
13330    // judge worktrees, so once the loop above has cleared all of them out,
13331    // the parent is a bare directory nobody else was ever going to remove -
13332    // git only ever managed what was inside it. Left alone, one of these
13333    // accumulates per fully-folded run; the operator's own machine had 74.
13334    // `remove_if_empty` re-checks rather than assuming: a run whose winner
13335    // was kept (`!drop_winner`) leaves its directory behind on purpose, and
13336    // so does anything a run never claimed that happens to share the bay.
13337    remove_if_empty(&root);
13338
13339    if state.enabled_worktree_config && drop_winner {
13340        // A release, not a raw disable: some sibling run in this repository
13341        // may still hold its own reference (see `git::acquire_worktree_config`),
13342        // and only the last release actually turns the setting back off.
13343        git::release_worktree_config(&repo).await.ok();
13344        state.enabled_worktree_config = false;
13345    }
13346    state.save_under(home)?;
13347    Ok(removed)
13348}
13349
13350async fn verify_merged_fold_sources(
13351    state: &RunState,
13352    drop_winner: bool,
13353    home: &Path,
13354) -> Result<Vec<std::fs::File>> {
13355    let winner = state.tally.as_ref().map(|t| t.winner);
13356    let mut heads = BTreeSet::new();
13357    for candidate in &state.candidates {
13358        if Some(candidate.label) == winner && !drop_winner {
13359            continue;
13360        }
13361        anyhow::ensure!(
13362            candidate.worktree.parent() == Some(state.worktree_root().as_path()),
13363            "candidate recovery source is outside the inventoried run root: {}",
13364            candidate.worktree.display()
13365        );
13366        match std::fs::symlink_metadata(&candidate.worktree) {
13367            Ok(_) => {}
13368            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13369            Err(error) => return Err(error.into()),
13370        }
13371        if !state.released_branches.contains(&candidate.branch)
13372            && git::branch_exists(&state.repo, &candidate.branch).await?
13373        {
13374            heads.insert(git::rev_parse(&state.repo, &candidate.branch).await?);
13375        }
13376    }
13377    let mut cargo_guards = Vec::new();
13378    let entries = match std::fs::read_dir(state.worktree_root()) {
13379        Ok(entries) => Some(entries),
13380        Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
13381        Err(error) => return Err(error.into()),
13382    };
13383    for entry in entries.into_iter().flatten() {
13384        let path = entry?.path();
13385        let metadata = std::fs::symlink_metadata(&path)?;
13386        anyhow::ensure!(
13387            metadata.is_dir() && !metadata.file_type().is_symlink(),
13388            "non-directory or symlink recovery source retained: {}",
13389            path.display()
13390        );
13391        if !drop_winner
13392            && winner.is_some_and(|w| {
13393                path.file_name()
13394                    .is_some_and(|name| name == format!("cand-{w}").as_str())
13395            })
13396        {
13397            continue;
13398        }
13399        anyhow::ensure!(
13400            path.join(".git").is_file(),
13401            "unregistered recovery source retained: {}",
13402            path.display()
13403        );
13404        anyhow::ensure!(
13405            git::git(
13406                &path,
13407                &[
13408                    "status",
13409                    "--porcelain",
13410                    "--untracked-files=all",
13411                    "--ignored",
13412                    "--",
13413                    ".",
13414                    ":(exclude)target"
13415                ]
13416            )
13417            .await?
13418            .is_empty(),
13419            "source changes or ignored recovery files retained: {}",
13420            path.display()
13421        );
13422        match std::fs::symlink_metadata(path.join("target")) {
13423            Ok(_) => {
13424                cargo_guards.extend(crate::retention::verify_residue(home, &state.id, &path).await?)
13425            }
13426            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
13427            Err(error) => return Err(error.into()),
13428        }
13429        for checkpoint in [
13430            "rebase-merge",
13431            "rebase-apply",
13432            "sequencer",
13433            "CHERRY_PICK_HEAD",
13434            "MERGE_HEAD",
13435            "REBASE_HEAD",
13436            "REVERT_HEAD",
13437            "BISECT_LOG",
13438            "AUTO_MERGE",
13439            "MERGE_MSG",
13440        ] {
13441            let location = git::git(
13442                &path,
13443                &[
13444                    "rev-parse",
13445                    "--path-format=absolute",
13446                    "--git-path",
13447                    checkpoint,
13448                ],
13449            )
13450            .await?;
13451            anyhow::ensure!(
13452                matches!(std::fs::symlink_metadata(location.trim()), Err(ref e)
13453                if e.kind() == std::io::ErrorKind::NotFound),
13454                "Git recovery checkpoint retained: {checkpoint}"
13455            );
13456        }
13457        heads.insert(git::rev_parse(&path, "HEAD").await?);
13458    }
13459    crate::fold_evidence::preserve(state, home, &heads).await?;
13460
13461    Ok(cargo_guards)
13462}
13463
13464/// Remove `dir` if it exists and has nothing in it.
13465///
13466/// Best-effort and silent by design: a directory that is not empty (a run
13467/// whose winner is still parked there, a stray file some other process left)
13468/// is exactly the case this must refuse, and a directory that is already gone
13469/// is not a failure worth reporting either. `std::fs::remove_dir` itself
13470/// already refuses a non-empty directory, so the emptiness check below is
13471/// belt, not suspenders - it is what keeps this from ever attempting the
13472/// removal in the case that matters, rather than trusting `remove_dir`'s
13473/// error path to have no side effects if it ever changed.
13474fn remove_if_empty(dir: &Path) {
13475    if dir.is_dir() && std::fs::read_dir(dir).is_ok_and(|mut entries| entries.next().is_none()) {
13476        std::fs::remove_dir(dir).ok();
13477    }
13478}
13479
13480/// Severity of the worst open finding in the last review round, for reporting.
13481pub fn worst_open(state: &RunState) -> Option<Severity> {
13482    state
13483        .reviews
13484        .last()?
13485        .reviews
13486        .iter()
13487        .flat_map(|r| r.findings.iter())
13488        .map(|f| f.severity)
13489        .max()
13490}
13491
13492/// `[worktree] setup` hides its products per worktree, which needs
13493/// `extensions.worktreeConfig`. Taken once per run and released with the run's
13494/// worktrees, like the hook's; every place that runs setup calls this first.
13495async fn ensure_setup_config(state: &mut RunState, repo: &Path) -> Result<()> {
13496    if !state.config.worktree.setup.is_empty() && !state.enabled_worktree_config {
13497        git::acquire_worktree_config(repo).await?;
13498        state.enabled_worktree_config = true;
13499    }
13500    Ok(())
13501}
13502
13503#[cfg(test)]
13504mod tests {
13505    #[tokio::test]
13506    async fn launcher_base_fetch_requires_remote_success() {
13507        let tmp = tempfile::tempdir().unwrap();
13508        let repo = tmp.path().join("repo");
13509        let remote = tmp.path().join("remote");
13510        std::fs::create_dir(&repo).unwrap();
13511        std::fs::create_dir(&remote).unwrap();
13512        git::git(&remote, &["init", "--bare", "-b", "main"])
13513            .await
13514            .unwrap();
13515        git::git(&repo, &["init", "-b", "main"]).await.unwrap();
13516        git::git(
13517            &repo,
13518            &[
13519                "-c",
13520                "user.name=Fixture",
13521                "-c",
13522                "user.email=fixture@example.invalid",
13523                "commit",
13524                "--allow-empty",
13525                "-m",
13526                "fixture",
13527            ],
13528        )
13529        .await
13530        .unwrap();
13531        git::git(
13532            &repo,
13533            &["remote", "add", "origin", remote.to_str().unwrap()],
13534        )
13535        .await
13536        .unwrap();
13537        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
13538        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
13539        assert_eq!(
13540            resolve_base(&repo, "main", "origin", None)
13541                .await
13542                .unwrap()
13543                .commit,
13544            head
13545        );
13546        #[cfg(unix)]
13547        {
13548            let policy: crate::config::Coordination = toml::from_str(
13549                "contract_file='fixture.contract'\ncontract_token='fixture-v1'\nlock_file='fixture.lock'",
13550            ).unwrap();
13551            std::fs::write(repo.join(".git/fixture.contract"), "fixture-v1").unwrap();
13552            std::fs::remove_file(repo.join(".git/FETCH_HEAD")).unwrap();
13553            let held = crate::repo_lock::SeatLease::operation_with(
13554                &repo,
13555                Some(&policy),
13556                &crate::repo_lock::WaitPolicy::default(),
13557            )
13558            .await
13559            .unwrap();
13560            assert!(held.is_some());
13561            let fetch = resolve_base(&repo, "main", "origin", Some(&policy));
13562            tokio::pin!(fetch);
13563            assert!(
13564                tokio::time::timeout(Duration::from_millis(50), &mut fetch)
13565                    .await
13566                    .is_err()
13567            );
13568            assert!(
13569                !repo.join(".git/FETCH_HEAD").exists(),
13570                "no fetch before admission"
13571            );
13572            drop(held);
13573            let fetched = fetch.await.unwrap();
13574            assert_eq!(fetched.commit, head);
13575            assert_eq!(fetched.remote, "origin");
13576            assert_eq!(fetched.branch, "main");
13577        }
13578        // A surviving tracking ref is not successful fetch evidence.
13579        git::git(
13580            &repo,
13581            &[
13582                "remote",
13583                "set-url",
13584                "origin",
13585                tmp.path().join("missing").to_str().unwrap(),
13586            ],
13587        )
13588        .await
13589        .unwrap();
13590        let error = resolve_base(&repo, "main", "origin", None)
13591            .await
13592            .unwrap_err()
13593            .to_string();
13594        assert!(error.contains("cannot read origin/main"));
13595        assert!(error.contains("refusing to branch off"));
13596    }
13597
13598    #[test]
13599    fn should_retitle_only_replaces_magi_shaped_or_leaked_titles() {
13600        let leaked = vec!["chore(deps): update a crate".to_owned()];
13601        let own = "fix(daemon): apply a chosen action";
13602        assert!(should_retitle("", own, &leaked));
13603        assert!(should_retitle(
13604            &format!("{REVIEW_PROMPT_OPENING} `x`"),
13605            own,
13606            &leaked
13607        ));
13608        assert!(should_retitle(
13609            "chore: land candidate A of run 1",
13610            own,
13611            &leaked
13612        ));
13613        assert!(should_retitle(
13614            "magi: candidate A (uncommitted work)",
13615            own,
13616            &leaked
13617        ));
13618        assert!(should_retitle("chore(deps): update a crate", own, &leaked));
13619        assert!(!should_retitle("feat: renamed by hand", own, &leaked));
13620        assert!(!should_retitle("", "chore(deps): update a crate", &leaked));
13621    }
13622
13623    #[test]
13624    fn pr_merge_plan_creates_adopts_or_stops() {
13625        assert_eq!(pr_merge_plan(Ok(land::OpenPr::None)), PrPlan::Create);
13626        assert_eq!(
13627            pr_merge_plan(Ok(land::OpenPr::One {
13628                url: "u".into(),
13629                title: "t".into()
13630            })),
13631            PrPlan::Adopt {
13632                url: "u".into(),
13633                title: "t".into()
13634            }
13635        );
13636        let PrPlan::Stop(many) =
13637            pr_merge_plan(Ok(land::OpenPr::Many(vec!["a".into(), "b".into()])))
13638        else {
13639            panic!("many must stop");
13640        };
13641        assert!(many.contains('a') && many.contains('b'));
13642        let PrPlan::Stop(err) = pr_merge_plan(Err(anyhow::anyhow!("bad token"))) else {
13643            panic!("a failed lookup must stop");
13644        };
13645        assert!(err.contains("bad token"));
13646    }
13647
13648    use super::*;
13649    use crate::run::GateStatus;
13650    use std::collections::BTreeMap;
13651    use std::time::Duration;
13652
13653    fn conductor() -> AgentSpec {
13654        AgentSpec {
13655            id: "conductor".to_owned(),
13656            kind: crate::config::AgentKind::Command,
13657            model: None,
13658            command: vec!["true".to_owned()],
13659            extra_args: Vec::new(),
13660            env: BTreeMap::new(),
13661            prompt_delivery: None,
13662            max_turn_secs: None,
13663        }
13664    }
13665
13666    fn spec(id: &str) -> AgentSpec {
13667        AgentSpec {
13668            id: id.to_owned(),
13669            kind: crate::config::AgentKind::Command,
13670            model: None,
13671            command: vec!["true".to_owned()],
13672            extra_args: Vec::new(),
13673            env: BTreeMap::new(),
13674            prompt_delivery: None,
13675            max_turn_secs: None,
13676        }
13677    }
13678
13679    fn ids(xs: &[&str]) -> BTreeSet<String> {
13680        xs.iter().map(|s| (*s).to_owned()).collect()
13681    }
13682
13683    #[test]
13684    fn pick_successor_skips_an_agent_another_seat_holds() {
13685        // Seats a and b of roster [a, b, c]; a fails, b holds the other seat.
13686        let roster = [spec("a"), spec("b"), spec("c")];
13687        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13688        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13689    }
13690
13691    #[test]
13692    fn pick_successor_respects_the_occupant_after_an_earlier_handover() {
13693        // The other seat started on c but was handed to d; c is free again.
13694        let roster = [spec("a"), spec("b"), spec("c"), spec("d")];
13695        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13696        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13697        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b", "c"]));
13698        assert_eq!(next.map(|s| s.id.as_str()), Some("d"));
13699    }
13700
13701    #[test]
13702    fn pick_successor_falls_back_to_a_duplicate_when_no_distinct_agent_remains() {
13703        let roster = [spec("a"), spec("b")];
13704        let next = pick_successor(&roster, 0, &ids(&["a"]), None, &ids(&["b"]));
13705        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13706        let carried = ids(&["b"]);
13707        let next = pick_successor(&roster, 0, &ids(&["a"]), Some(&carried), &ids(&["b"]));
13708        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13709    }
13710
13711    #[test]
13712    fn pick_successor_returns_none_without_any_untried_successor() {
13713        let roster = [spec("a"), spec("b")];
13714        assert!(pick_successor(&roster, 1, &ids(&["b"]), None, &ids(&["a"])).is_none());
13715        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), None, &ids(&[])).is_none());
13716        let carried = ids(&["a"]);
13717        assert!(pick_successor(&roster, 0, &ids(&["a", "b"]), Some(&carried), &ids(&[])).is_none());
13718    }
13719
13720    #[test]
13721    fn pick_successor_rescue_avoids_another_seats_occupant() {
13722        // b is a carried failure and free; a is held by the other seat.
13723        let roster = [spec("a"), spec("b"), spec("c")];
13724        let carried = ids(&["b", "c"]);
13725        let next = pick_successor(&roster, 2, &ids(&["c"]), Some(&carried), &ids(&["a"]));
13726        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13727    }
13728
13729    #[test]
13730    fn next_for_seat_prefers_an_agent_that_has_not_failed() {
13731        let roster = [spec("a"), spec("b"), spec("c")];
13732        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["b"]));
13733        assert_eq!(next.map(|s| s.id.as_str()), Some("c"));
13734    }
13735
13736    #[test]
13737    fn next_for_seat_rescues_a_failed_agent_only_when_nothing_else_is_left() {
13738        let roster = [spec("a"), spec("b"), spec("c")];
13739        let failed = ids(&["a", "b", "c"]);
13740        // Rescue looks at the whole roster, once per id, then runs out.
13741        let mut tried = ids(&["b"]);
13742        let first = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13743        assert_eq!(first.id, "a");
13744        tried.insert(first.id.clone());
13745        let second = next_for_seat(&roster, 1, &tried, &failed).expect("rescue");
13746        assert_eq!(second.id, "c");
13747        tried.insert(second.id.clone());
13748        assert!(next_for_seat(&roster, 1, &tried, &failed).is_none());
13749    }
13750
13751    #[test]
13752    fn next_for_seat_ignores_failed_ids_no_longer_on_the_roster() {
13753        let roster = [spec("a"), spec("b")];
13754        let next = next_for_seat(&roster, 0, &ids(&["a"]), &ids(&["gone"]));
13755        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13756    }
13757
13758    #[test]
13759    fn pick_start_spec_starts_on_the_last_answerer_when_still_eligible() {
13760        let roster = [spec("a"), spec("b"), spec("c")];
13761        let h = SeatHistory {
13762            failed: ids(&["a"]),
13763            last_ok: Some("b".to_owned()),
13764            last_fail: None,
13765        };
13766        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&h)).id, "b");
13767        // A last answerer that left the roster, or later failed, is ignored.
13768        let gone = SeatHistory {
13769            last_ok: Some("zzz".to_owned()),
13770            ..h.clone()
13771        };
13772        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&gone)).id, "b");
13773        let failed = SeatHistory {
13774            failed: ids(&["a", "b"]),
13775            last_ok: Some("b".to_owned()),
13776            last_fail: None,
13777        };
13778        assert_eq!(pick_start_spec(&roster, spec("a"), Some(&failed)).id, "c");
13779    }
13780
13781    #[test]
13782    fn handover_seat_mints_a_session_id_distinct_from_the_previous_agents() {
13783        let first = SeatState::new("review-1", "alpha", 7);
13784        let next = handover_seat("review-1", "gamma", 7);
13785        assert_ne!(first.claude_session, next.claude_session);
13786    }
13787
13788    #[test]
13789    fn re_handing_a_seat_to_the_same_agent_mints_a_new_session_id() {
13790        let mut state = state_with_summary("x", "y");
13791        let a = handover_seat("review-1", "beta", state.next_seat_seed());
13792        let b = handover_seat("review-1", "beta", state.next_seat_seed());
13793        assert_ne!(a.claude_session, b.claude_session);
13794    }
13795
13796    #[test]
13797    fn pick_start_spec_falls_back_to_the_spec_when_the_whole_roster_failed() {
13798        let roster = [spec("a"), spec("b")];
13799        let h = SeatHistory {
13800            failed: ids(&["a", "b"]),
13801            ..SeatHistory::default()
13802        };
13803        assert_eq!(pick_start_spec(&roster, spec("b"), Some(&h)).id, "b");
13804        assert_eq!(pick_start_spec(&roster, spec("b"), None).id, "b");
13805        assert_eq!(pick_start_spec(&[], spec("b"), Some(&h)).id, "b");
13806    }
13807
13808    // `next_untried_in_roster` is the property `resume_seat_handovers`'s own
13809    // fallback loop depends on to terminate: it must walk forward from the
13810    // seat's own position, never restart at the front of the roster, and it
13811    // must never hand back an id already tried, however many times that id
13812    // happens to appear.
13813
13814    #[test]
13815    fn failure_signature_ignores_numbers_and_paths() {
13816        assert_eq!(
13817            failure_signature("exited with Some(2) and no usable output"),
13818            failure_signature("exited with Some(137) and no usable output")
13819        );
13820        assert_eq!(
13821            failure_signature("process exited with no usable output (exit code 2)"),
13822            failure_signature("process exited with no usable output (exit code 137)")
13823        );
13824        for (old, new) in [
13825            (
13826                "exited with Some(2) and no usable output",
13827                "process exited with no usable output (exit code 2)",
13828            ),
13829            (
13830                "exited with None and no usable output",
13831                "process exited with no usable output (exit code unavailable)",
13832            ),
13833        ] {
13834            let previous = FailClass::Other(failure_signature(old));
13835            let current = FailClass::Other(failure_signature(new));
13836            assert_eq!(previous, current, "resumption retains the failure class");
13837            assert!(!should_hand_over(Some(&previous), &current));
13838        }
13839        assert_eq!(
13840            failure_signature("cannot open /tmp/a/b.txt: denied\nsecond line"),
13841            failure_signature("cannot open /var/x.txt: denied")
13842        );
13843        assert_ne!(failure_signature("boom"), failure_signature("bang"));
13844    }
13845
13846    #[test]
13847    fn quota_and_timeout_always_hand_over_other_failures_stop_on_a_repeat() {
13848        let other = FailClass::Other("x".into());
13849        assert!(should_hand_over(None, &FailClass::Quota));
13850        assert!(should_hand_over(Some(&other), &FailClass::Quota));
13851        assert!(should_hand_over(
13852            Some(&FailClass::Timeout),
13853            &FailClass::Timeout
13854        ));
13855        assert!(should_hand_over(None, &other));
13856        assert!(!should_hand_over(Some(&other), &other));
13857        assert!(should_hand_over(
13858            Some(&other),
13859            &FailClass::Other("y".into())
13860        ));
13861        // A quota or timeout in between ends the run of identical failures.
13862        assert!(should_hand_over(Some(&FailClass::Timeout), &other));
13863        assert!(should_hand_over(Some(&FailClass::Quota), &other));
13864    }
13865
13866    #[test]
13867    fn a_handover_seat_never_reuses_the_previous_agents_session_id() {
13868        let a = SeatState::new("judge-1", "alpha", 7);
13869        let b = handover_seat("judge-1", "beta", 7);
13870        assert_ne!(a.claude_session, b.claude_session);
13871        assert_eq!(b.turns, 0);
13872    }
13873
13874    #[test]
13875    fn a_timeout_is_classified_apart_from_other_failures() {
13876        assert_eq!(
13877            FailClass::of(&AgentOutcome::Failed(TIMED_OUT.to_owned())),
13878            Some(FailClass::Timeout)
13879        );
13880        assert!(matches!(
13881            FailClass::of(&AgentOutcome::Failed("boom".to_owned())),
13882            Some(FailClass::Other(_))
13883        ));
13884    }
13885
13886    #[test]
13887    fn next_untried_in_roster_walks_forward_from_the_seats_own_position() {
13888        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13889        let tried = BTreeSet::from(["beta".to_owned()]);
13890        // beta sits at index 1; the next candidate is gamma, never alpha —
13891        // which is very likely a different candidate slot's own agent.
13892        let next = next_untried_in_roster(&roster, 1, &tried);
13893        assert_eq!(next.map(|s| s.id.as_str()), Some("gamma"));
13894    }
13895
13896    #[test]
13897    fn next_untried_in_roster_does_not_wrap_back_past_its_own_start() {
13898        let roster = vec![spec("alpha"), spec("beta")];
13899        let tried = BTreeSet::from(["beta".to_owned()]);
13900        // beta is the roster's last entry: nothing follows it, and alpha —
13901        // earlier in the roster, almost certainly a different candidate
13902        // slot's own agent — must not be reached by wrapping back to it.
13903        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13904    }
13905
13906    #[test]
13907    fn next_untried_in_roster_stops_once_the_tail_is_exhausted_even_if_earlier_ids_are_untried() {
13908        let roster = vec![spec("alpha"), spec("beta"), spec("gamma")];
13909        let tried = BTreeSet::from(["beta".to_owned(), "gamma".to_owned()]);
13910        // beta (index 1) and gamma (index 2, the only entry after it) have
13911        // both been tried; alpha (index 0) never has, but it comes before
13912        // beta's own position, so there is nothing further for this seat.
13913        assert!(next_untried_in_roster(&roster, 1, &tried).is_none());
13914    }
13915
13916    #[test]
13917    fn next_untried_in_roster_skips_ids_already_tried_even_when_duplicated() {
13918        let roster = vec![spec("a"), spec("a"), spec("b")];
13919        let tried = BTreeSet::from(["a".to_owned()]);
13920        let next = next_untried_in_roster(&roster, 0, &tried);
13921        assert_eq!(next.map(|s| s.id.as_str()), Some("b"));
13922    }
13923
13924    #[test]
13925    fn next_untried_in_roster_returns_none_once_every_id_is_tried() {
13926        let roster = vec![spec("a"), spec("b")];
13927        let tried = BTreeSet::from(["a".to_owned(), "b".to_owned()]);
13928        assert!(next_untried_in_roster(&roster, 0, &tried).is_none());
13929    }
13930
13931    #[test]
13932    fn remove_if_empty_only_ever_takes_a_bare_directory() {
13933        let dir = tempfile::tempdir().unwrap();
13934        let bay = dir.path().join("ffff");
13935
13936        // Not there yet: nothing to do, nothing to panic on.
13937        remove_if_empty(&bay);
13938        assert!(!bay.exists());
13939
13940        // Something still inside - the winner's worktree, or a stray file -
13941        // keeps the directory standing.
13942        std::fs::create_dir_all(bay.join("cand-A")).unwrap();
13943        remove_if_empty(&bay);
13944        assert!(bay.exists(), "non-empty directory must survive");
13945
13946        // Once the last entry is gone, so is the directory itself.
13947        std::fs::remove_dir(bay.join("cand-A")).unwrap();
13948        remove_if_empty(&bay);
13949        assert!(!bay.exists(), "an empty bay is a leftover, not a record");
13950    }
13951
13952    // `round_is_clean` is the exact decision this task fixed: a round with a
13953    // seat that never answered must not read the same as a round every seat
13954    // actually reviewed. These are deterministic and process-free by design —
13955    // the equivalent end-to-end check (a real reviewer timing out under a
13956    // live graph run) is a genuine race against wall-clock contention, and a
13957    // spawn slow enough to blow even a generous budget under a loaded test
13958    // run must not turn this specific regression check flaky.
13959
13960    #[test]
13961    fn a_full_panel_that_found_nothing_is_clean() {
13962        assert!(round_is_clean(
13963            0,
13964            true,
13965            2,
13966            2,
13967            0,
13968            IncompleteReviewPolicy::Block
13969        ));
13970    }
13971
13972    #[test]
13973    fn a_missing_seat_is_never_clean_under_the_default_policy() {
13974        assert!(!round_is_clean(
13975            0,
13976            true,
13977            1,
13978            2,
13979            0,
13980            IncompleteReviewPolicy::Block
13981        ));
13982    }
13983
13984    #[test]
13985    fn warn_policy_still_refuses_a_missing_seat_with_open_findings() {
13986        assert!(!round_is_clean(
13987            1,
13988            true,
13989            1,
13990            2,
13991            0,
13992            IncompleteReviewPolicy::Warn
13993        ));
13994    }
13995
13996    #[test]
13997    fn warn_policy_gates_a_missing_seat_once_what_answered_is_clean() {
13998        assert!(round_is_clean(
13999            0,
14000            true,
14001            1,
14002            2,
14003            0,
14004            IncompleteReviewPolicy::Warn
14005        ));
14006    }
14007
14008    #[test]
14009    fn a_full_panel_with_an_open_finding_is_not_clean() {
14010        assert!(!round_is_clean(
14011            1,
14012            true,
14013            2,
14014            2,
14015            0,
14016            IncompleteReviewPolicy::Block
14017        ));
14018    }
14019
14020    #[test]
14021    fn a_full_panel_with_a_red_e2e_is_not_clean() {
14022        assert!(!round_is_clean(
14023            0,
14024            false,
14025            2,
14026            2,
14027            0,
14028            IncompleteReviewPolicy::Block
14029        ));
14030    }
14031
14032    // The stall this task closes: under the default `block` policy, a seat
14033    // missing only because it was rate limited must not force a wait for a
14034    // session limit that will not lift by the next round. `round_is_clean`
14035    // is where that quorum carve-out lives; the review loop around it never
14036    // changes what a reviewer's vote or a finding's severity means.
14037
14038    #[test]
14039    fn a_seat_missing_only_to_its_own_quota_is_clean_under_the_default_policy() {
14040        // 1 of 2 answered, and the one missing was quota'd — the exact
14041        // "review-2 rate limited (quota)" shape from the field report.
14042        assert!(round_is_clean(
14043            0,
14044            true,
14045            1,
14046            2,
14047            1,
14048            IncompleteReviewPolicy::Block
14049        ));
14050    }
14051
14052    #[test]
14053    fn a_seat_missing_for_a_reason_other_than_quota_still_waits() {
14054        // 1 of 2 answered, but the miss was a crash/timeout/parse failure,
14055        // not a quota loss (`quota_missing` stays 0) — worth another try.
14056        assert!(!round_is_clean(
14057            0,
14058            true,
14059            1,
14060            2,
14061            0,
14062            IncompleteReviewPolicy::Block
14063        ));
14064    }
14065
14066    #[test]
14067    fn a_quota_loss_does_not_excuse_an_open_finding_or_a_red_e2e() {
14068        assert!(!round_is_clean(
14069            1,
14070            true,
14071            1,
14072            2,
14073            1,
14074            IncompleteReviewPolicy::Block
14075        ));
14076        assert!(!round_is_clean(
14077            0,
14078            false,
14079            1,
14080            2,
14081            1,
14082            IncompleteReviewPolicy::Block
14083        ));
14084    }
14085
14086    #[test]
14087    fn a_panel_lost_entirely_to_quota_still_waits_rather_than_deciding_on_nobody() {
14088        // Every seat quota'd, nobody answered: there is no panel to decide
14089        // on, so this must fall through to the existing block-and-retry
14090        // fallback rather than call an unreviewed patch clean.
14091        assert!(!round_is_clean(
14092            0,
14093            true,
14094            0,
14095            2,
14096            2,
14097            IncompleteReviewPolicy::Block
14098        ));
14099    }
14100
14101    fn outcome(code: Option<i32>, resource_blocked: bool) -> CommandOutcome {
14102        CommandOutcome {
14103            command: "test".to_owned(),
14104            code,
14105            output_tail: String::new(),
14106            duration_ms: 0,
14107            resource_blocked,
14108        }
14109    }
14110
14111    #[cfg(unix)]
14112    #[tokio::test]
14113    async fn incremental_admission_is_a_resource_hold_without_a_build_retry() {
14114        ask_test_home();
14115        let temp = tempfile::tempdir().unwrap();
14116        let mut runner = runner_at(RunStatus::Gating);
14117        let shell = vec!["sh".to_owned(), "-c".to_owned()];
14118        let commands = vec![
14119            "printf '%s\\n' 'incremental cache admission blocked: budget exhausted' 'error: could not compile fixture'; exit 101".to_owned(),
14120            "touch must-not-run".to_owned(),
14121        ];
14122        let (outcomes, retried, pids) = run_e2e_with_retry(
14123            &mut runner.state,
14124            &shell,
14125            &commands,
14126            temp.path(),
14127            Duration::from_secs(5),
14128            "fixture",
14129        )
14130        .await;
14131        assert_eq!(outcomes.len(), 1);
14132        assert!(outcomes[0].resource_blocked);
14133        assert!(!outcomes[0].build_failed());
14134        assert!(!retried);
14135        assert!(pids.is_empty());
14136        assert!(!temp.path().join("must-not-run").exists());
14137        assert_eq!(
14138            runner.state.resource_block.as_ref().unwrap().kind,
14139            "incremental cache admission"
14140        );
14141    }
14142
14143    #[test]
14144    fn verify_is_inconclusive_only_when_a_resource_blocked_outcome_is_present() {
14145        assert!(!verify_inconclusive(&[outcome(Some(0), false)]));
14146        assert!(
14147            !verify_inconclusive(&[outcome(Some(1), false)]),
14148            "an ordinary failure is still evidence about the patch"
14149        );
14150        assert!(verify_inconclusive(&[outcome(None, true)]));
14151        assert!(
14152            verify_inconclusive(&[outcome(Some(0), false), outcome(None, true)]),
14153            "one inconclusive outcome taints the whole batch"
14154        );
14155        assert!(!verify_inconclusive(&[]));
14156    }
14157
14158    #[tokio::test]
14159    async fn timed_out_pid_waiting_returns_as_soon_as_every_pid_is_confirmed_dead() {
14160        // Alive for the first two checks, then dead - confirms the loop
14161        // actually re-polls rather than deciding once and sleeping out the
14162        // ceiling regardless.
14163        let calls = std::sync::atomic::AtomicUsize::new(0);
14164        let started = Instant::now();
14165        wait_for_pids_with(
14166            &[123],
14167            |_| calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) < 2,
14168            Duration::from_millis(5),
14169            Duration::from_secs(5),
14170        )
14171        .await;
14172        assert!(
14173            calls.load(std::sync::atomic::Ordering::SeqCst) >= 3,
14174            "must keep checking rather than deciding on the first answer"
14175        );
14176        assert!(
14177            started.elapsed() < Duration::from_secs(1),
14178            "must return the moment it is confirmed dead, not wait out the ceiling"
14179        );
14180    }
14181
14182    #[tokio::test]
14183    async fn timed_out_pid_waiting_gives_up_at_its_ceiling_if_never_confirmed_dead() {
14184        let started = Instant::now();
14185        wait_for_pids_with(
14186            &[123],
14187            |_| true, // never reports dead
14188            Duration::from_millis(5),
14189            Duration::from_millis(30),
14190        )
14191        .await;
14192        let elapsed = started.elapsed();
14193        assert!(
14194            elapsed >= Duration::from_millis(30),
14195            "must not give up before its own ceiling: {elapsed:?}"
14196        );
14197        assert!(
14198            elapsed < Duration::from_secs(1),
14199            "must not wait past its own ceiling either: {elapsed:?}"
14200        );
14201    }
14202
14203    #[tokio::test]
14204    async fn timed_out_pid_waiting_is_a_no_op_when_nothing_was_still_running() {
14205        let started = Instant::now();
14206        wait_for_pids_with(
14207            &[],
14208            |_| true,
14209            Duration::from_secs(5),
14210            Duration::from_secs(5),
14211        )
14212        .await;
14213        assert!(
14214            started.elapsed() < Duration::from_millis(200),
14215            "an empty pid list has nothing to confirm"
14216        );
14217    }
14218
14219    // `review_conclusion` is the exact decision the review hand-off task
14220    // fixed: a round budget spent (or a tree that stopped moving) must not
14221    // collapse into `Blocked` regardless of what verification actually
14222    // said. Deterministic and process-free for the same reason the
14223    // `round_is_clean` family above is.
14224    fn review_round(
14225        clean: bool,
14226        blocking: usize,
14227        answered: usize,
14228        expected: usize,
14229        progressed: bool,
14230        e2e_ok: bool,
14231    ) -> ReviewRound {
14232        ReviewRound {
14233            round: 1,
14234            head: "h".to_owned(),
14235            verified_head: None,
14236            verified_at: None,
14237            reviews: Vec::new(),
14238            e2e: vec![CommandOutcome {
14239                command: "test".to_owned(),
14240                code: Some(if e2e_ok { 0 } else { 1 }),
14241                output_tail: String::new(),
14242                duration_ms: 0,
14243                resource_blocked: false,
14244            }],
14245            verify_retried: false,
14246            e2e_deferred: false,
14247            e2e_defer_reason: None,
14248            fix: None,
14249            blocking,
14250            answered,
14251            expected,
14252            clean,
14253            progressed,
14254            vote_split: false,
14255            reconsideration: Vec::new(),
14256            verdict: None,
14257        }
14258    }
14259
14260    #[test]
14261    fn review_conclusion_is_none_when_nothing_has_run() {
14262        assert_eq!(review_conclusion(&[], 3), None);
14263    }
14264
14265    #[test]
14266    fn review_conclusion_is_none_while_rounds_remain() {
14267        let rounds = vec![review_round(false, 1, 2, 2, true, true)];
14268        assert_eq!(review_conclusion(&rounds, 3), None);
14269    }
14270
14271    #[test]
14272    fn review_conclusion_is_gating_once_a_round_is_clean() {
14273        let rounds = vec![review_round(true, 0, 2, 2, false, true)];
14274        assert_eq!(review_conclusion(&rounds, 3), Some(RunStatus::Gating));
14275    }
14276
14277    #[test]
14278    fn review_conclusion_hands_off_when_the_budget_is_spent_and_e2e_is_green() {
14279        let rounds = vec![
14280            review_round(false, 1, 2, 2, true, true),
14281            review_round(false, 1, 2, 2, true, true),
14282        ];
14283        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Gating));
14284    }
14285
14286    #[test]
14287    fn review_conclusion_blocks_when_the_budget_is_spent_and_e2e_is_red() {
14288        let rounds = vec![
14289            review_round(false, 1, 2, 2, true, true),
14290            review_round(false, 1, 2, 2, true, false),
14291        ];
14292        assert_eq!(review_conclusion(&rounds, 2), Some(RunStatus::Blocked));
14293    }
14294
14295    #[test]
14296    fn review_conclusion_stays_none_when_the_budget_is_spent_but_the_last_round_could_not_run() {
14297        // Magi never got a command to run against this round's own head — a
14298        // resource-blocked attempt, not a red one — so this must never
14299        // settle on `Blocked` the way a genuine e2e failure would. `None`
14300        // here is what tells `Runner::review_loop` to retry the check
14301        // itself rather than trust this cheap recomputation with a verdict
14302        // it cannot actually produce.
14303        let mut blocked = review_round(false, 1, 2, 2, true, false);
14304        blocked.e2e[0].resource_blocked = true;
14305        let rounds = vec![review_round(false, 1, 2, 2, true, true), blocked];
14306        assert_eq!(review_conclusion(&rounds, 2), None);
14307    }
14308
14309    #[test]
14310    fn review_conclusion_blocks_an_incomplete_panel_that_raised_nothing_even_with_green_e2e() {
14311        // Missing input, not a verified tree — never a hand-off candidate.
14312        let rounds = vec![review_round(false, 0, 1, 2, false, true)];
14313        assert_eq!(review_conclusion(&rounds, 1), Some(RunStatus::Blocked));
14314    }
14315
14316    #[test]
14317    fn review_conclusion_hands_off_when_the_tree_stagnates_before_the_budget_is_spent() {
14318        let rounds = vec![
14319            review_round(false, 1, 2, 2, false, true),
14320            review_round(false, 1, 2, 2, false, true),
14321        ];
14322        assert_eq!(review_conclusion(&rounds, 10), Some(RunStatus::Gating));
14323    }
14324
14325    fn secs(n: u64) -> Duration {
14326        Duration::from_secs(n)
14327    }
14328
14329    /// A throwaway repo with one commit on `main`, for tests that need `merge`
14330    /// to make real (and, if it runs at all, real*ly fail*) git calls.
14331    #[tokio::test]
14332    async fn selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync() {
14333        const CHILD: &str = "MAGI_SELECTED_PUBLIC_ANCESTRY_TEST";
14334        if std::env::var_os(CHILD).is_none() {
14335            // Other tests reconcile every run in the process-global home,
14336            // briefly taking owner locks. This ancestry test expects an idle
14337            // predecessor, so give it its own process and pinned test home.
14338            let output = std::process::Command::new(std::env::current_exe().unwrap())
14339                .args([
14340                    "--exact",
14341                    "graph::tests::selected_public_ancestry_survives_obsolete_candidate_and_current_base_sync",
14342                    "--nocapture",
14343                ])
14344                .env(CHILD, "1")
14345                .env_remove("MAGI_WORKER_HOME")
14346                .env_remove("MAGI_WORKER_ID")
14347                .output()
14348                .unwrap();
14349            assert!(
14350                output.status.success()
14351                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
14352                "{}\n{}",
14353                String::from_utf8_lossy(&output.stdout),
14354                String::from_utf8_lossy(&output.stderr)
14355            );
14356            return;
14357        }
14358        ask_test_home();
14359        let tmp = tempfile::tempdir().unwrap();
14360        let repo = tmp.path().join("repo");
14361        std::fs::create_dir_all(&repo).unwrap();
14362        init_repo(&repo);
14363        let g = |args: &[&str]| {
14364            let out = std::process::Command::new("git")
14365                .args(args)
14366                .current_dir(&repo)
14367                .quiet()
14368                .output()
14369                .unwrap();
14370            assert!(
14371                out.status.success(),
14372                "{}",
14373                String::from_utf8_lossy(&out.stderr)
14374            );
14375        };
14376        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
14377        g(&["checkout", "-b", "historical"]);
14378        std::fs::write(repo.join("implementation"), "incompatible alternative").unwrap();
14379        g(&["add", "."]);
14380        g(&["commit", "-m", "old alternative"]);
14381        let obsolete = git::rev_parse(&repo, "HEAD").await.unwrap();
14382        g(&["checkout", "-b", "magi/task-selected", &base]);
14383        std::fs::write(repo.join("infrastructure"), "prerequisite").unwrap();
14384        g(&["add", "."]);
14385        g(&["commit", "-m", "measurement infrastructure"]);
14386        let prerequisite = git::rev_parse(&repo, "HEAD").await.unwrap();
14387        std::fs::write(repo.join("implementation"), "measured implementation").unwrap();
14388        g(&["add", "."]);
14389        g(&["commit", "-m", "dependent implementation"]);
14390        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
14391        g(&["checkout", "main"]);
14392        std::fs::write(repo.join("unrelated"), "current main change").unwrap();
14393        g(&["add", "."]);
14394        g(&["commit", "-m", "unrelated base change"]);
14395        let current = git::rev_parse(&repo, "HEAD").await.unwrap();
14396        let origin = tmp.path().join("origin.git");
14397        g(&[
14398            "clone",
14399            "--bare",
14400            repo.to_str().unwrap(),
14401            origin.to_str().unwrap(),
14402        ]);
14403        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
14404        g(&["fetch", "origin"]);
14405        let selection = crate::pr_recovery::Selection {
14406            url: "https://example.test/project/pull/1".into(),
14407            branch: "magi/task-selected".into(),
14408            head: head.clone(),
14409            source_head: None,
14410            source_run: None,
14411            source_proof: None,
14412            disposition: Some(crate::pr_recovery::Disposition::Review),
14413            instruction: "preserve measured behavior".into(),
14414            approved_at: jiff::Timestamp::now(),
14415        };
14416        let mut template = runner_at(RunStatus::Reviewing);
14417        template.state.repo = repo.clone();
14418        template.state.base_commit = current.clone();
14419        template.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
14420        template.state.origin = Some(crate::run::Origin::queue("selected"));
14421        template.state.pr_head = Some(selection.branch.clone());
14422        template.state.pr_recovery = Some(selection.clone());
14423        let mut runner =
14424            Runner::open_selected(&repo, template.state, template.roles, 1, current, selection)
14425                .await
14426                .unwrap();
14427        let wt = runner.state.winner().unwrap().worktree.clone();
14428        assert_eq!(git::rev_parse(&wt, "HEAD").await.unwrap(), head);
14429        assert!(git::is_ancestor(&wt, &prerequisite, "HEAD").await);
14430        assert!(!git::is_ancestor(&wt, &obsolete, "HEAD").await);
14431        assert!(runner.state.seeds.is_empty());
14432        assert_eq!(runner.state.instruction, "preserve measured behavior");
14433        runner.sync_to_base().await.unwrap();
14434        assert_eq!(
14435            std::fs::read_to_string(wt.join("infrastructure")).unwrap(),
14436            "prerequisite"
14437        );
14438        assert_eq!(
14439            std::fs::read_to_string(wt.join("implementation")).unwrap(),
14440            "measured implementation"
14441        );
14442        assert_eq!(
14443            std::fs::read_to_string(wt.join("unrelated")).unwrap(),
14444            "current main change"
14445        );
14446        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14447        assert_eq!(
14448            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14449            head
14450        );
14451
14452        // A failed selected-source setup stays inspectable, never resumable as a new competition.
14453        let mut partial = runner_at(RunStatus::Reviewing);
14454        partial.state.repo = repo.clone();
14455        partial.state.config.graph.worktree_root = Some(tmp.path().join("failed-worktrees"));
14456        partial.state.config.worktree.setup = vec![crate::config::SetupStep {
14457            run: Some("exit 1".into()),
14458            ..Default::default()
14459        }];
14460        let selection = runner.state.pr_recovery.clone().unwrap();
14461        partial.state.pr_recovery = Some(selection.clone());
14462        partial.state.pr_head = Some(selection.branch.clone());
14463        partial.state.origin = Some(crate::run::Origin::queue("selected"));
14464        let id = partial.state.id.clone();
14465        let error = match Runner::open_selected(
14466            &repo,
14467            partial.state,
14468            partial.roles,
14469            1,
14470            runner.state.base_commit.clone(),
14471            selection,
14472        )
14473        .await
14474        {
14475            Ok(_) => panic!("setup must fail"),
14476            Err(e) => e,
14477        };
14478        assert_eq!(
14479            error
14480                .downcast_ref::<crate::run::StartupFailure>()
14481                .unwrap()
14482                .run,
14483            id
14484        );
14485        let failed = RunState::load(&id).unwrap();
14486        assert_eq!(failed.status, RunStatus::Failed);
14487        assert_eq!(failed.liveness(false), crate::run::Liveness::Dead);
14488        assert!(Runner::resume(&id).is_err());
14489        runner.state.status = RunStatus::Ready;
14490        runner.state.driver_exited = true;
14491        runner.state.save().unwrap();
14492        let mut retry = runner_at(RunStatus::Reviewing);
14493        retry.state.repo = repo.clone();
14494        retry.state.config.graph.worktree_root = Some(tmp.path().join("retry-worktrees"));
14495        let mut reselected = runner.state.pr_recovery.clone().unwrap();
14496        reselected.approved_at = jiff::Timestamp::now();
14497        retry.state.origin = Some(crate::run::Origin::queue("selected"));
14498        retry.state.pr_head = Some(reselected.branch.clone());
14499        retry.state.pr_recovery = Some(reselected.clone());
14500        crate::pr_head::check_predecessors(&retry.state, &reselected.branch)
14501            .await
14502            .unwrap();
14503        let restarted = Runner::open_selected(
14504            &repo,
14505            retry.state,
14506            retry.roles,
14507            1,
14508            runner.state.base_commit.clone(),
14509            reselected,
14510        )
14511        .await
14512        .unwrap();
14513        assert_eq!(
14514            git::rev_parse(&restarted.state.winner().unwrap().worktree, "HEAD")
14515                .await
14516                .unwrap(),
14517            head
14518        );
14519        assert_eq!(git::rev_parse(&repo, "historical").await.unwrap(), obsolete);
14520        assert_eq!(
14521            git::rev_parse(&repo, "magi/task-selected").await.unwrap(),
14522            head
14523        );
14524    }
14525
14526    #[tokio::test]
14527    #[cfg(unix)]
14528    async fn fallback_and_interrupted_restoration_require_coordination() {
14529        use std::os::fd::AsRawFd as _;
14530        ask_test_home();
14531        let tmp = tempfile::tempdir().unwrap();
14532        let mut runner = registered_runner(tmp.path()).await;
14533        runner.state.config.graph.timeout_resource_wait = 1;
14534        let winner = runner.state.candidates[0].clone();
14535        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
14536        let leftover = winner.worktree.join("leftover");
14537        std::fs::write(&leftover, "untrusted edits").unwrap();
14538        let next = tmp.path().join("next-fixer");
14539        let mut failed = spec("first");
14540        failed.command = vec!["sh".into(), "-c".into(), "exit 1".into()];
14541        let mut successor = spec("second");
14542        successor.command = vec![
14543            "sh".into(),
14544            "-c".into(),
14545            format!("touch '{}'; echo done", next.display()),
14546        ];
14547        let attempts = vec![
14548            (failed.clone(), "first".into()),
14549            (successor.clone(), "second".into()),
14550        ];
14551        let lock_path = runner.state.repo.join(".git/updater.lock");
14552        let external = std::fs::File::create(&lock_path).unwrap();
14553        assert_eq!(
14554            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14555            0
14556        );
14557        let artifacts = tmp.path().join("artifacts");
14558        let build = |spec, seat| SeatJob {
14559            spec,
14560            seat,
14561            cwd: winner.worktree.clone(),
14562            prompt: "fixture".into(),
14563            timeout: Duration::from_secs(10),
14564            allow_write: true,
14565            sessions: false,
14566            artifacts: artifacts.clone(),
14567            stem: "fix".into(),
14568            handover: None,
14569        };
14570        let (_, _, out) = runner
14571            .ask_fixer_among(
14572                attempts.clone(),
14573                "graft",
14574                None,
14575                Some((&winner.worktree, &head)),
14576                build,
14577            )
14578            .await;
14579        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
14580        assert!(leftover.exists(), "contention must prevent clean");
14581        assert!(!next.exists());
14582        assert!(runner.state.handovers.is_empty());
14583        // Early interrupted-repair rollback uses the same protected route.
14584        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14585            from_run: "earlier".into(),
14586            generation: 1,
14587            mode: crate::recovery::Mode::VerifyRepair,
14588            reasons: Vec::new(),
14589            reviewed_head: None,
14590            rounds: Vec::new(),
14591            open_findings: Vec::new(),
14592            evidence: None,
14593            criteria_inherited: true,
14594            repair: crate::recovery::Repair::Started {
14595                before: head.clone(),
14596            },
14597        });
14598        assert!(
14599            runner
14600                .recovery_repair(&winner)
14601                .await
14602                .unwrap_err()
14603                .is::<crate::repo_lock::ResourceBlocked>()
14604        );
14605        assert!(matches!(
14606            runner.state.recovery.as_ref().unwrap().repair,
14607            crate::recovery::Repair::Started { .. }
14608        ));
14609        assert!(leftover.exists());
14610        drop(external);
14611        runner.recovery_repair(&winner).await.unwrap();
14612        assert!(!leftover.exists());
14613        assert!(matches!(
14614            runner.state.recovery.as_ref().unwrap().repair,
14615            crate::recovery::Repair::Done { .. }
14616        ));
14617        // A hard lock-open failure also leaves HEAD and untracked work intact.
14618        std::fs::remove_file(&lock_path).unwrap();
14619        std::fs::create_dir(&lock_path).unwrap();
14620        std::fs::write(&leftover, "untrusted edits").unwrap();
14621        let (_, _, out) = runner
14622            .ask_fixer_among(
14623                attempts,
14624                "graft",
14625                None,
14626                Some((&winner.worktree, &head)),
14627                build,
14628            )
14629            .await;
14630        assert!(matches!(out, AgentOutcome::Failed(_)));
14631        assert!(leftover.exists());
14632        assert!(!next.exists());
14633        assert_eq!(
14634            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
14635            head
14636        );
14637    }
14638
14639    #[cfg(unix)]
14640    async fn registered_runner(root: &Path) -> Runner {
14641        repository_runner(root, crate::config::test_coordination()).await
14642    }
14643
14644    async fn repository_runner(
14645        root: &Path,
14646        coordination: Option<crate::config::Coordination>,
14647    ) -> Runner {
14648        ask_test_home();
14649        let repo = root.join("repo");
14650        std::fs::create_dir_all(&repo).unwrap();
14651        init_repo(&repo);
14652        if coordination.is_some() {
14653            std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
14654        }
14655        let mut runner = runner_at(RunStatus::Prep);
14656        runner.state.repo = repo.clone();
14657        runner.state.config.coordination = coordination;
14658        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
14659        runner.state.config.graph.worktree_root = Some(root.join("wt"));
14660        runner.state.config.graph.advise = false;
14661        runner.roles.implementers = vec![spec("implement")];
14662        runner.roles.judges = vec![spec("judge")];
14663        runner.prep().await.unwrap();
14664        runner
14665    }
14666
14667    const AUTH_OUTAGE: &str = r#"{"is_error":true,"terminal_reason":"api_error","result":"Failed to authenticate: OAuth session expired and could not be refreshed"}"#;
14668
14669    #[tokio::test]
14670    async fn all_unauthenticated_wave_stops_the_node_without_nudges() {
14671        let tmp = tempfile::tempdir().unwrap();
14672        let mut runner = repository_runner(tmp.path(), None).await;
14673        let prompts = runner.state.config.prompts.clone();
14674        let run = runner.state.id.clone();
14675        let ctx = WaveCtx {
14676            carry_seats: false,
14677            pauses: None,
14678            run: &run,
14679            node: "advise",
14680            prompts: &prompts,
14681            cache: None,
14682            round: None,
14683        };
14684        let jobs: Vec<_> = ["alpha", "beta"]
14685            .into_iter()
14686            .map(|id| SeatJob {
14687                spec: scripted(id, AUTH_OUTAGE),
14688                seat: SeatState::new(id, id, 7),
14689                prompt: "propose".into(),
14690                cwd: runner.state.candidates[0].worktree.clone(),
14691                timeout: Duration::from_secs(10),
14692                allow_write: false,
14693                sessions: false,
14694                artifacts: agent::artifacts_dir(&runner.state.dir()),
14695                stem: id.into(),
14696                handover: None,
14697            })
14698            .collect();
14699        let mut losses = Vec::new();
14700        let results = ask_wave_with(
14701            jobs.clone(),
14702            runner.sem.clone(),
14703            2,
14704            &[],
14705            &ctx,
14706            &mut losses,
14707            &mut runner.state,
14708            &|_: &str| Ok(()),
14709        )
14710        .await;
14711        assert!(
14712            results
14713                .iter()
14714                .all(|(seat, result, nudges)| seat.turns == 1 && result.is_err() && *nudges == 0)
14715        );
14716        assert_eq!(runner.state.quota.len(), 2);
14717        assert!(
14718            runner
14719                .state
14720                .quota
14721                .iter()
14722                .all(|q| q.cause == agent::QuotaCause::Authentication && !q.agent.is_empty())
14723        );
14724        assert!(!runner.state.advise_attempted);
14725        runner.stop_for_resource().unwrap();
14726        assert_eq!(runner.state.status, RunStatus::Blocked);
14727        assert_eq!(runner.state.resource_block.as_ref().unwrap().node, "advise");
14728        assert_eq!(
14729            runner.state.resource_block.as_ref().unwrap().kind,
14730            "agent authentication"
14731        );
14732        // A machine loss followed by an ordinary failure is not evidence
14733        // that the whole node was unavailable, and must not earn a refund.
14734        runner.state.resource_block = None;
14735        runner.state.quota.clear();
14736        let mut mixed = jobs;
14737        mixed[1].spec.command = vec!["false".into()];
14738        let results = ask_wave_with(
14739            mixed,
14740            runner.sem.clone(),
14741            0,
14742            &[],
14743            &ctx,
14744            &mut losses,
14745            &mut runner.state,
14746            &|_: &str| Ok(()),
14747        )
14748        .await;
14749        assert!(results.iter().all(|(_, res, _)| res.is_err()));
14750        assert!(runner.state.resource_block.is_none());
14751        assert_eq!(losses.len(), 1);
14752    }
14753
14754    #[tokio::test]
14755    async fn unauthenticated_implementation_keeps_its_candidate_pending() {
14756        let tmp = tempfile::tempdir().unwrap();
14757        let mut runner = repository_runner(tmp.path(), None).await;
14758        let implementer = scripted("implement", AUTH_OUTAGE);
14759        runner.roles.implementers = vec![implementer.clone()];
14760        runner.state.config.agents = vec![implementer];
14761        runner.implement().await.unwrap();
14762        assert_eq!(runner.state.status, RunStatus::Blocked);
14763        assert_eq!(runner.state.quota.len(), 1);
14764        assert_eq!(
14765            runner.state.quota[0].cause,
14766            agent::QuotaCause::Authentication
14767        );
14768        let candidate = &runner.state.candidates[0];
14769        assert!(candidate.failed.is_none() && !candidate.empty && candidate.commits == 0);
14770        assert_eq!(runner.state.seats["impl-A"].turns, 1);
14771    }
14772
14773    #[tokio::test]
14774    #[cfg(unix)]
14775    async fn parked_implementer_persists_completed_work() {
14776        for registered in [false, true] {
14777            let tmp = tempfile::tempdir().unwrap();
14778            let mut runner = registered_runner(tmp.path()).await;
14779            if !registered {
14780                std::fs::remove_file(runner.state.repo.join(".git/updater.contract")).unwrap();
14781            }
14782            let marker = tmp.path().join("started");
14783            let release = tmp.path().join("release");
14784            let mut implementer = spec("implement");
14785            implementer.command = vec![
14786                "sh".into(),
14787                "-c".into(),
14788                format!(
14789                    "touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; echo completed > completed; echo saved-result",
14790                    marker.display(),
14791                    release.display()
14792                ),
14793            ];
14794            runner.roles.implementers = vec![implementer.clone()];
14795            runner.state.config.agents = vec![implementer];
14796            let pause = runner.pause.clone();
14797            let (result, ()) = tokio::join!(runner.implement(), async {
14798                tokio::time::timeout(Duration::from_secs(10), async {
14799                    while !marker.exists() {
14800                        tokio::time::sleep(Duration::from_millis(10)).await;
14801                    }
14802                })
14803                .await
14804                .unwrap();
14805                pause.park();
14806                std::fs::write(&release, "go").unwrap();
14807            });
14808            result.unwrap();
14809            let candidate = &runner.state.candidates[0];
14810            assert!(candidate.commits > 0, "{candidate:?}");
14811            assert!(candidate.summary.contains("saved-result"));
14812            assert_eq!(candidate.files, 1);
14813            let before = candidate.commits;
14814            runner.implement().await.unwrap();
14815            assert_eq!(runner.state.candidates[0].commits, before);
14816        }
14817    }
14818
14819    #[tokio::test]
14820    #[cfg(unix)]
14821    async fn resource_blocked_recovery_retries_the_unstarted_fixer() {
14822        let tmp = tempfile::tempdir().unwrap();
14823        let mut runner = registered_runner(tmp.path()).await;
14824        let winner = runner.state.candidates[0].clone();
14825        let cache = tmp.path().join("cache");
14826        runner.state.config.verify.e2e = vec![format!("CARGO_TARGET_DIR={} true", cache.display())];
14827        runner.state.config.graph.timeout_resource_wait = 0;
14828        let owner = crate::cache::Owner::here("other", "e2e", "e2e", &cache, "abc");
14829        let crate::cache::AcquireOutcome::Acquired(held) =
14830            crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap()
14831        else {
14832            panic!("cache must be free")
14833        };
14834        runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14835            from_run: "earlier".into(),
14836            generation: 1,
14837            mode: crate::recovery::Mode::VerifyRepair,
14838            reasons: Vec::new(),
14839            reviewed_head: None,
14840            rounds: Vec::new(),
14841            open_findings: Vec::new(),
14842            evidence: None,
14843            criteria_inherited: true,
14844            repair: crate::recovery::Repair::NotStarted,
14845        });
14846        runner.recovery_repair(&winner).await.unwrap();
14847        assert!(matches!(
14848            runner.state.recovery.as_ref().unwrap().repair,
14849            crate::recovery::Repair::WaitingForRestore { .. }
14850        ));
14851        drop(held);
14852        runner.state.resource_block = None;
14853        runner.state.config.graph.timeout_resource_wait = 1;
14854        runner.recovery_repair(&winner).await.unwrap();
14855        assert!(matches!(
14856            runner.state.recovery.as_ref().unwrap().repair,
14857            crate::recovery::Repair::Done { .. }
14858        ));
14859        assert!(runner.state.seats.values().any(|seat| seat.turns > 0));
14860    }
14861
14862    #[tokio::test]
14863    #[cfg(unix)]
14864    async fn repository_blocked_graft_and_recovery_retry_after_deferred_restore() {
14865        use std::os::fd::AsRawFd as _;
14866        for graft in [false, true] {
14867            let tmp = tempfile::tempdir().unwrap();
14868            let mut runner = registered_runner(tmp.path()).await;
14869            runner.state.config.graph.timeout_resource_wait = 1;
14870            let winner = runner.state.candidates[0].clone();
14871            let marker = tmp.path().join("fixer-started");
14872            let mut fixer = spec("fixer");
14873            fixer.command = vec![
14874                "sh".into(),
14875                "-c".into(),
14876                format!("echo started >> '{}'; echo done", marker.display()),
14877            ];
14878            runner.roles.fixer = Some(vec![fixer]);
14879            if graft {
14880                let mut loser = winner.clone();
14881                loser.label = 'B';
14882                loser.index = 1;
14883                runner.state.candidates.push(loser);
14884                runner.state.config.graph.graft_runner_up = true;
14885                runner.state.tally = Some(Tally {
14886                    first_choice: BTreeMap::new(),
14887                    borda: BTreeMap::new(),
14888                    winner: winner.label,
14889                    rankings: 1,
14890                    unanimous_initial: true,
14891                    deliberated: false,
14892                    changed_votes: 0,
14893                    unanimous_final: true,
14894                    tie_break: None,
14895                    judges: 1,
14896                    present: 1,
14897                    quorum: 1,
14898                    met_quorum: true,
14899                    uncontested: None,
14900                });
14901                runner.state.judgements.push(crate::run::Judgement {
14902                    judge: 1,
14903                    seat: "judge-1".into(),
14904                    agent: "judge".into(),
14905                    ranking: vec![winner.label, 'B'],
14906                    reasons: BTreeMap::new(),
14907                    confidence: None,
14908                    strengths: BTreeMap::from([("B".into(), vec!["handles empty input".into()])]),
14909                    order: vec![0, 1],
14910                    failed: None,
14911                    duration_ms: 0,
14912                });
14913            } else {
14914                runner.state.recovery = Some(crate::recovery::RecoveryRecord {
14915                    from_run: "earlier".into(),
14916                    generation: 1,
14917                    mode: crate::recovery::Mode::VerifyRepair,
14918                    reasons: Vec::new(),
14919                    reviewed_head: None,
14920                    rounds: Vec::new(),
14921                    open_findings: Vec::new(),
14922                    evidence: None,
14923                    criteria_inherited: true,
14924                    repair: crate::recovery::Repair::NotStarted,
14925                });
14926            }
14927            let reader = crate::repo_lock::SeatLease::for_checkout(
14928                &winner.worktree,
14929                false,
14930                crate::config::test_coordination().as_ref(),
14931            )
14932            .await
14933            .unwrap();
14934            let pause = runner.pause.clone();
14935            let (result, ()) = tokio::join!(
14936                async {
14937                    if graft {
14938                        runner.graft().await
14939                    } else {
14940                        runner.recovery_repair(&winner).await
14941                    }
14942                },
14943                async {
14944                    tokio::time::sleep(Duration::from_millis(100)).await;
14945                    pause.park();
14946                }
14947            );
14948            result.unwrap();
14949            assert!(!marker.exists(), "a blocked seat must not spawn");
14950            // Exercise the persisted state, as a binary replacement/resume does.
14951            runner.state = RunState::load(&runner.state.id).unwrap();
14952            if graft {
14953                assert_eq!(
14954                    runner.state.graft.as_ref().unwrap().outcome,
14955                    GraftOutcome::WaitingForRestore
14956                );
14957            } else {
14958                assert!(matches!(
14959                    runner.state.recovery.as_ref().unwrap().repair,
14960                    crate::recovery::Repair::WaitingForRestore { .. }
14961                ));
14962            }
14963            drop(reader);
14964            runner.pause = Pause::new();
14965            runner.state.resource_block = None;
14966            // Deferred restoration must still respect an external lock owner.
14967            let external =
14968                std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
14969            assert_eq!(
14970                unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
14971                0
14972            );
14973            let leftover = winner.worktree.join("leftover");
14974            std::fs::write(&leftover, "earlier fallback edits").unwrap();
14975            let result = if graft {
14976                runner.graft().await
14977            } else {
14978                runner.recovery_repair(&winner).await
14979            };
14980            assert!(
14981                result
14982                    .unwrap_err()
14983                    .is::<crate::repo_lock::ResourceBlocked>()
14984            );
14985            assert!(leftover.exists());
14986            assert!(!marker.exists());
14987            drop(external);
14988            runner.state.resource_block = None;
14989            let result = if graft {
14990                runner.graft().await
14991            } else {
14992                runner.recovery_repair(&winner).await
14993            };
14994            result.unwrap();
14995            assert!(!leftover.exists(), "restore must precede retry");
14996            assert_eq!(std::fs::read_to_string(&marker).unwrap(), "started\n");
14997            if graft {
14998                assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
14999            } else {
15000                assert!(matches!(
15001                    runner.state.recovery.as_ref().unwrap().repair,
15002                    crate::recovery::Repair::Done { .. }
15003                ));
15004            }
15005        }
15006    }
15007
15008    #[tokio::test]
15009    #[cfg(unix)]
15010    async fn interrupted_graft_restore_waits_and_preserves_its_record() {
15011        use std::os::fd::AsRawFd as _;
15012        let tmp = tempfile::tempdir().unwrap();
15013        let mut runner = registered_runner(tmp.path()).await;
15014        runner.state.config.graph.timeout_resource_wait = 1;
15015        let winner = runner.state.candidates[0].clone();
15016        let pre = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15017        runner.state.tally = Some(Tally {
15018            first_choice: BTreeMap::new(),
15019            borda: BTreeMap::new(),
15020            winner: winner.label,
15021            rankings: 0,
15022            unanimous_initial: false,
15023            deliberated: false,
15024            changed_votes: 0,
15025            unanimous_final: false,
15026            tie_break: None,
15027            judges: 0,
15028            present: 0,
15029            quorum: 0,
15030            met_quorum: true,
15031            uncontested: Some("fixture".into()),
15032        });
15033        runner.state.graft = Some(GraftRecord {
15034            pre: pre.clone(),
15035            post: None,
15036            lines: None,
15037            limit: None,
15038            agent: None,
15039            outcome: GraftOutcome::Running,
15040            requested: Vec::new(),
15041            adoption: None,
15042            adoption_error: None,
15043        });
15044        std::fs::write(winner.worktree.join("README.md"), "unfinished graft\n").unwrap();
15045        git::commit_all(&winner.worktree, "unfinished graft")
15046            .await
15047            .unwrap();
15048        let unfinished = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15049        let leftover = winner.worktree.join("leftover");
15050        std::fs::write(&leftover, "untrusted edits").unwrap();
15051        let path = runner.state.repo.join(".git/updater.lock");
15052        let external = std::fs::File::create(&path).unwrap();
15053        assert_eq!(
15054            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15055            0
15056        );
15057        assert!(
15058            runner
15059                .graft()
15060                .await
15061                .unwrap_err()
15062                .is::<crate::repo_lock::ResourceBlocked>()
15063        );
15064        assert!(matches!(
15065            runner.state.graft.as_ref().unwrap().outcome,
15066            GraftOutcome::Running
15067        ));
15068        assert_eq!(
15069            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15070            unfinished
15071        );
15072        assert!(leftover.exists());
15073        drop(external);
15074        std::fs::remove_file(&path).unwrap();
15075        std::fs::create_dir(&path).unwrap();
15076        assert!(runner.graft().await.is_err());
15077        assert!(matches!(
15078            runner.state.graft.as_ref().unwrap().outcome,
15079            GraftOutcome::Running
15080        ));
15081        assert_eq!(
15082            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15083            unfinished
15084        );
15085        assert!(leftover.exists());
15086        std::fs::remove_dir(&path).unwrap();
15087        runner.graft().await.unwrap();
15088        assert!(runner.state.graft.as_ref().unwrap().outcome.settled());
15089        assert_eq!(git::rev_parse(&winner.worktree, "HEAD").await.unwrap(), pre);
15090        assert!(!leftover.exists());
15091    }
15092
15093    #[tokio::test]
15094    #[cfg(unix)]
15095    async fn repository_contention_blocks_and_parks_without_poisoning_preparation() {
15096        use std::os::fd::AsRawFd as _;
15097        let tmp = tempfile::tempdir().unwrap();
15098        let mut runner = registered_runner(tmp.path()).await;
15099        runner.state.prep_complete = Some(false);
15100        runner.state.config.graph.timeout_resource_wait = 1;
15101        runner.state.save().unwrap();
15102        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15103        assert_eq!(
15104            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15105            0
15106        );
15107        runner.execute().await.unwrap();
15108        let disk = RunState::load(&runner.state.id).unwrap();
15109        assert!(disk.driver_exited);
15110        assert_eq!(disk.driver_pid, None);
15111        assert_eq!(runner.state.status, RunStatus::Blocked);
15112        assert_eq!(
15113            runner.state.resource_block.as_ref().unwrap().kind,
15114            "repository coordination"
15115        );
15116        assert!(runner.state.prep_failure.is_none());
15117        assert!(runner.state.seats.is_empty());
15118        runner.pause.park();
15119        // Include headroom for Git subprocesses under parallel test load;
15120        // the repository contention deadline itself remains one second.
15121        tokio::time::timeout(Duration::from_secs(10), runner.execute())
15122            .await
15123            .unwrap()
15124            .unwrap();
15125        assert!(runner.state.parked);
15126        assert!(runner.state.prep_failure.is_none());
15127        let disk = RunState::load(&runner.state.id).unwrap();
15128        assert!(disk.driver_exited);
15129        assert_eq!(disk.driver_pid, None);
15130    }
15131
15132    #[tokio::test]
15133    #[cfg(unix)]
15134    async fn dropping_execute_during_a_repository_wait_releases_the_driver() {
15135        use std::os::fd::AsRawFd as _;
15136        let tmp = tempfile::tempdir().unwrap();
15137        let mut runner = registered_runner(tmp.path()).await;
15138        runner.state.prep_complete = Some(false);
15139        runner.state.config.graph.timeout_resource_wait = 30;
15140        runner.state.save().unwrap();
15141        let external = std::fs::File::create(runner.state.repo.join(".git/updater.lock")).unwrap();
15142        assert_eq!(
15143            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15144            0
15145        );
15146        assert!(
15147            tokio::time::timeout(Duration::from_millis(100), runner.execute())
15148                .await
15149                .is_err()
15150        );
15151        let disk = RunState::load(&runner.state.id).unwrap();
15152        assert!(disk.driver_exited);
15153        assert_eq!(disk.driver_pid, None);
15154        assert!(disk.status.resumable());
15155        assert!(runner.state.driver_exited);
15156        assert_eq!(runner.state.driver_pid, None);
15157        runner.state.save().unwrap();
15158        assert_eq!(RunState::load(&runner.state.id).unwrap().driver_pid, None);
15159    }
15160
15161    #[tokio::test]
15162    #[cfg(unix)]
15163    async fn gate_takes_cache_before_repository_so_an_editing_wave_can_finish() {
15164        let tmp = tempfile::tempdir().unwrap();
15165        let mut runner = registered_runner(tmp.path()).await;
15166        let winner = runner.state.candidates[0].clone();
15167        let cache = tmp.path().join("cache");
15168        let started = tmp.path().join("pre-gate-started");
15169        runner.state.config.verify.pre_gate = vec![format!("touch '{}'", started.display())];
15170        runner.state.config.verify.gate =
15171            vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
15172        let home = crate::run::home();
15173        let owner =
15174            crate::cache::Owner::here("editing-wave", "implement", "*", &winner.worktree, "HEAD");
15175        let guard = match crate::cache::try_acquire(&home, &cache, &owner).unwrap() {
15176            crate::cache::AcquireOutcome::Acquired(g) => g,
15177            crate::cache::AcquireOutcome::Busy(busy) => panic!("cache must be free: {busy:?}"),
15178        };
15179        let mut gate = Box::pin(runner.run_gate(&winner));
15180        assert!(
15181            tokio::time::timeout(Duration::from_millis(150), &mut gate)
15182                .await
15183                .is_err()
15184        );
15185        assert!(!started.exists(), "pre-gate must wait for the cache too");
15186        // Simulate the wave that already owns the cache entering invoke.
15187        // It must obtain the editor reservation and finish without waiting
15188        // for the gate that is queued behind its cache.
15189        let wait = crate::repo_lock::WaitPolicy::new(Duration::from_millis(200), || false);
15190        let editor = crate::repo_lock::SeatLease::checkout_with(
15191            &winner.worktree,
15192            true,
15193            crate::config::test_coordination().as_ref(),
15194            &wait,
15195        )
15196        .await
15197        .unwrap();
15198        drop(editor);
15199        guard.release();
15200        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15201            .await
15202            .unwrap()
15203            .unwrap();
15204        drop(gate);
15205        assert!(outcomes.iter().all(CommandOutcome::ok));
15206        assert!(started.exists());
15207    }
15208
15209    #[tokio::test]
15210    #[cfg(unix)]
15211    async fn post_seat_rescue_and_gate_wait_for_shared_lock() {
15212        use std::os::fd::AsRawFd as _;
15213
15214        ask_test_home();
15215        let tmp = tempfile::tempdir().unwrap();
15216        let repo = tmp.path().join("repo");
15217        std::fs::create_dir_all(&repo).unwrap();
15218        init_repo(&repo);
15219        let common = repo.join(".git");
15220        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15221        let mut runner = runner_at(RunStatus::Prep);
15222        runner.state.repo = repo.clone();
15223        runner.state.config.coordination = crate::config::test_coordination();
15224        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15225        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15226        runner.state.config.graph.advise = false;
15227        runner.roles.implementers = vec![spec("implement")];
15228        runner.roles.judges = vec![spec("judge")];
15229        runner.prep().await.unwrap();
15230        let winner = runner.state.candidates[0].clone();
15231        // Emulate edits already left by a completed seat. The fixture CLI
15232        // only reports completion; it performs no repository operations.
15233        std::fs::write(winner.worktree.join("leftover"), "seat work\n").unwrap();
15234        let completed = tmp.path().join("seat-completed");
15235        runner.roles.implementers[0].command = vec![
15236            "sh".into(),
15237            "-c".into(),
15238            format!("touch '{}'; echo completed", completed.display()),
15239        ];
15240        let outsider = std::fs::File::create(common.join("updater.lock")).unwrap();
15241        assert_eq!(
15242            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15243            0
15244        );
15245        let before = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15246        let mut implement = Box::pin(runner.implement());
15247        tokio::select! {
15248            result = &mut implement => panic!("rescue must wait: {result:?}"),
15249            result = tokio::time::timeout(Duration::from_secs(10), async {
15250                while !completed.exists() {
15251                    tokio::time::sleep(Duration::from_millis(20)).await;
15252                }
15253            }) => result.unwrap(),
15254        }
15255        assert!(
15256            tokio::time::timeout(Duration::from_millis(200), &mut implement)
15257                .await
15258                .is_err()
15259        );
15260        assert_eq!(
15261            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15262            before
15263        );
15264        assert_eq!(
15265            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15266            0
15267        );
15268        tokio::time::timeout(Duration::from_secs(10), &mut implement)
15269            .await
15270            .unwrap()
15271            .unwrap();
15272        drop(implement);
15273        assert_ne!(
15274            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15275            before
15276        );
15277        assert!(git::is_clean(&winner.worktree).await.unwrap());
15278
15279        let started = tmp.path().join("pre-gate-started");
15280        let proceed = tmp.path().join("pre-gate-proceed");
15281        let checked = tmp.path().join("gate-ran");
15282        runner.state.config.verify.pre_gate = vec![format!(
15283            "echo mechanical >> README.md; touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15284            started.display(),
15285            proceed.display()
15286        )];
15287        runner.state.config.verify.gate = vec![format!("touch '{}'", checked.display())];
15288        assert_eq!(
15289            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15290            0
15291        );
15292        let mut gate = Box::pin(runner.run_gate(&winner));
15293        assert!(
15294            tokio::time::timeout(Duration::from_millis(200), &mut gate)
15295                .await
15296                .is_err()
15297        );
15298        assert!(!started.exists());
15299        assert!(!checked.exists());
15300        assert!(git::is_clean(&winner.worktree).await.unwrap());
15301        assert_eq!(
15302            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_UN) },
15303            0
15304        );
15305        tokio::select! {
15306            result = &mut gate => panic!("pre-gate must wait for signal: {result:?}"),
15307            result = tokio::time::timeout(Duration::from_secs(10), async {
15308                while !started.exists() {
15309                    tokio::time::sleep(Duration::from_millis(20)).await;
15310                }
15311            }) => result.unwrap(),
15312        }
15313        assert_eq!(
15314            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15315            -1,
15316            "mechanical edits must hold the real lock"
15317        );
15318        std::fs::write(proceed, "ready").unwrap();
15319        let outcomes = tokio::time::timeout(Duration::from_secs(10), &mut gate)
15320            .await
15321            .unwrap()
15322            .unwrap();
15323        drop(gate);
15324        assert!(outcomes.iter().all(CommandOutcome::ok));
15325        assert!(checked.exists());
15326        assert!(runner.state.pre_gate_commit.is_some());
15327        assert!(git::is_clean(&winner.worktree).await.unwrap());
15328        assert_eq!(
15329            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15330            0,
15331            "gate completion must release coordination"
15332        );
15333        drop(outsider);
15334        std::fs::remove_file(common.join("updater.lock")).unwrap();
15335        std::fs::create_dir(common.join("updater.lock")).unwrap();
15336        std::fs::remove_file(&started).unwrap();
15337        std::fs::remove_file(&checked).unwrap();
15338        let head = git::rev_parse(&winner.worktree, "HEAD").await.unwrap();
15339        let error = runner.run_gate(&winner).await.unwrap_err();
15340        assert!(
15341            error
15342                .to_string()
15343                .contains("acquire mandatory repository lock")
15344        );
15345        assert!(
15346            !started.exists(),
15347            "failed acquisition must prevent pre-gate"
15348        );
15349        assert!(!checked.exists(), "failed acquisition must prevent gate");
15350        assert!(git::is_clean(&winner.worktree).await.unwrap());
15351        assert_eq!(
15352            git::rev_parse(&winner.worktree, "HEAD").await.unwrap(),
15353            head
15354        );
15355    }
15356
15357    #[tokio::test]
15358    #[cfg(unix)]
15359    async fn preparation_waits_for_shared_lock_before_checkout_and_holds_it_through_setup() {
15360        use std::os::fd::AsRawFd as _;
15361        use std::time::Duration;
15362
15363        ask_test_home();
15364        let tmp = tempfile::tempdir().unwrap();
15365        let repo = tmp.path().join("repo");
15366        std::fs::create_dir_all(&repo).unwrap();
15367        init_repo(&repo);
15368        let common = repo.join(".git");
15369        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
15370        let lock = std::fs::File::create(common.join("updater.lock")).unwrap();
15371        assert_eq!(
15372            unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15373            0
15374        );
15375        let mut runner = runner_at(RunStatus::Prep);
15376        runner.state.repo = repo.clone();
15377        runner.state.config.coordination = crate::config::test_coordination();
15378        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15379        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15380        runner.state.config.graph.advise = true;
15381        runner.state.config.graph.advisors = 1;
15382        runner.roles.implementers = vec![spec("implement")];
15383        runner.roles.judges = vec![spec("judge")];
15384        let root = runner.state.worktree_root();
15385        let started = tmp.path().join("setup-started");
15386        let proceed = tmp.path().join("setup-proceed");
15387        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15388            run: Some(format!(
15389                "touch '{}'; while ! test -f '{}'; do sleep 0.02; done",
15390                started.display(),
15391                proceed.display()
15392            )),
15393            ..Default::default()
15394        }];
15395        let branches = git::git(&repo, &["branch", "--list"]).await.unwrap();
15396        let config = git::git(&repo, &["config", "--local", "--list"])
15397            .await
15398            .unwrap();
15399        let mut prep = Box::pin(runner.prep());
15400        assert!(
15401            tokio::time::timeout(Duration::from_millis(200), &mut prep)
15402                .await
15403                .is_err()
15404        );
15405        assert!(
15406            !root.exists(),
15407            "contention must prevent even checkout creation"
15408        );
15409        assert!(!started.exists(), "contention must prevent setup");
15410        assert_eq!(
15411            git::git(&repo, &["branch", "--list"]).await.unwrap(),
15412            branches
15413        );
15414        assert_eq!(
15415            git::git(&repo, &["config", "--local", "--list"])
15416                .await
15417                .unwrap(),
15418            config
15419        );
15420        drop(lock);
15421        tokio::select! {
15422            result = &mut prep => panic!("setup must wait for its signal: {result:?}"),
15423            result = tokio::time::timeout(Duration::from_secs(10), async {
15424                while !started.exists() {
15425                    tokio::time::sleep(Duration::from_millis(20)).await;
15426                }
15427            }) => result.unwrap(),
15428        }
15429        let outsider = std::fs::File::open(common.join("updater.lock")).unwrap();
15430        assert_eq!(
15431            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15432            -1
15433        );
15434        assert!(
15435            tokio::time::timeout(
15436                Duration::from_millis(100),
15437                crate::repo_lock::SeatLease::for_checkout(
15438                    &repo,
15439                    true,
15440                    crate::config::test_coordination().as_ref()
15441                )
15442            )
15443            .await
15444            .is_err(),
15445            "preparation must also exclude local editors"
15446        );
15447        std::fs::write(proceed, "ready").unwrap();
15448        tokio::time::timeout(Duration::from_secs(10), &mut prep)
15449            .await
15450            .unwrap()
15451            .unwrap();
15452        drop(prep);
15453        assert_eq!(runner.state.prep_complete, Some(true));
15454        assert!(root.join("judge-1").exists());
15455        assert!(root.join("advisor-1").exists());
15456        assert_eq!(
15457            unsafe { libc::flock(outsider.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
15458            0
15459        );
15460    }
15461
15462    #[tokio::test]
15463    #[cfg(unix)]
15464    async fn preparation_lock_failure_prevents_repository_changes() {
15465        ask_test_home();
15466        let tmp = tempfile::tempdir().unwrap();
15467        let repo = tmp.path().join("repo");
15468        std::fs::create_dir_all(&repo).unwrap();
15469        init_repo(&repo);
15470        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
15471        // A directory at the required lock path makes opening the real lock
15472        // fail, rather than merely exercising an invalid registration.
15473        std::fs::create_dir(repo.join(".git/updater.lock")).unwrap();
15474        let mut runner = runner_at(RunStatus::Prep);
15475        runner.state.repo = repo.clone();
15476        runner.state.config.coordination = crate::config::test_coordination();
15477        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15478        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15479        runner.roles.implementers = vec![spec("implement")];
15480        let error = runner.prep().await.unwrap_err();
15481        assert!(
15482            error
15483                .to_string()
15484                .contains("acquire mandatory repository lock")
15485        );
15486        assert!(!runner.state.worktree_root().exists());
15487        assert!(runner.state.candidates.is_empty());
15488        assert!(!repo.join(".git/config.worktree").exists());
15489        assert_eq!(
15490            git::git(&repo, &["branch", "--list"]).await.unwrap().trim(),
15491            "* main"
15492        );
15493    }
15494
15495    #[tokio::test]
15496    async fn preparation_records_creation_intent_before_git_can_fail() {
15497        ask_test_home();
15498        let tmp = tempfile::tempdir().unwrap();
15499        let repo = tmp.path().join("repo");
15500        std::fs::create_dir_all(&repo).unwrap();
15501        init_repo(&repo);
15502        let mut runner = runner_at(RunStatus::Prep);
15503        runner.state.repo = repo.clone();
15504        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15505        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15506        runner.state.config.blind.commit_msg_hook = false;
15507        runner.roles.implementers = vec![spec("one")];
15508        let label = blind::assign_labels(1, runner.state.seed)[0];
15509        let branch = runner.state.branch_for(label);
15510        // A conflicting branch makes Git refuse creation before a candidate
15511        // can be registered. The durable intent must still identify it.
15512        git::git(&repo, &["branch", &branch]).await.unwrap();
15513        assert!(runner.prep().await.is_err());
15514        let saved = RunState::load(&runner.state.id).unwrap();
15515        assert!(saved.candidates.is_empty());
15516        let event = saved
15517            .events
15518            .iter()
15519            .find(|e| e.node == "prep-intent")
15520            .unwrap();
15521        let intent: serde_json::Value = serde_json::from_str(&event.message).unwrap();
15522        assert_eq!(intent["branch"], branch);
15523        assert_eq!(intent["start"], runner.state.base_commit);
15524        assert_eq!(
15525            intent["worktree"],
15526            runner
15527                .state
15528                .worktree_root()
15529                .join(format!("cand-{label}"))
15530                .to_string_lossy()
15531                .as_ref()
15532        );
15533        assert_eq!(intent["picks"], serde_json::json!([]));
15534    }
15535
15536    #[tokio::test]
15537    async fn invalid_pinned_sources_create_no_partial_worktrees() {
15538        ask_test_home();
15539        let tmp = tempfile::tempdir().unwrap();
15540        let repo = tmp.path().join("repo");
15541        std::fs::create_dir_all(&repo).unwrap();
15542        init_repo(&repo);
15543        let mut runner = runner_at(RunStatus::Prep);
15544        runner.state.repo = repo.clone();
15545        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15546        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
15547        runner.state.sources.pick_commits.push(refs::Input {
15548            token: "missing".into(),
15549            sha: "0".repeat(40),
15550            provenance: refs::Provenance::ExplicitCli,
15551        });
15552        assert!(runner.prep().await.is_err());
15553        assert!(runner.state.candidates.is_empty());
15554        assert!(!runner.state.worktree_root().exists());
15555    }
15556
15557    #[tokio::test]
15558    async fn prose_and_legacy_inference_never_import_into_candidates() {
15559        ask_test_home();
15560        let tmp = tempfile::tempdir().unwrap();
15561        let repo = tmp.path().join("repo");
15562        std::fs::create_dir_all(&repo).unwrap();
15563        init_repo(&repo);
15564        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15565        git::git(&repo, &["checkout", "-b", "magi/history/A"])
15566            .await
15567            .unwrap();
15568        std::fs::write(repo.join("unwanted"), "do not import").unwrap();
15569        git::git(&repo, &["add", "."]).await.unwrap();
15570        git::git(&repo, &["commit", "-m", "historical implementation"])
15571            .await
15572            .unwrap();
15573        let source = git::rev_parse(&repo, "HEAD").await.unwrap();
15574        git::git(&repo, &["checkout", "main"]).await.unwrap();
15575        for instruction in [
15576            format!("Example: magi/history/A at {source}"),
15577            format!("Quoted failure: cannot apply referenced commit {source} from magi/history/A"),
15578            format!("Never import magi/history/A or {source}"),
15579            format!("Conversation answer: I mentioned {source} only as background"),
15580        ] {
15581            let mut runner = runner_at(RunStatus::Prep);
15582            runner.state.repo = repo.clone();
15583            runner.state.base_commit = base.clone();
15584            runner.state.instruction = instruction.clone();
15585            runner.state.config.graph.worktree_root = Some(tmp.path().join(&runner.state.id));
15586            runner.state.config.blind.commit_msg_hook = false;
15587            runner.state.seeds = vec![refs::Seed {
15588                token: "magi/history/A".into(),
15589                sha: source.clone(),
15590                kind: refs::SeedKind::Unmerged,
15591                branch: true,
15592                detail: String::new(),
15593            }];
15594            runner.roles.implementers = vec![spec("one")];
15595            assert_eq!(seeded_instruction(&runner.state), instruction);
15596            // Model an old record with inferred seeds but no structured input field.
15597            let mut legacy = serde_json::to_value(&runner.state).unwrap();
15598            legacy.as_object_mut().unwrap().remove("sources");
15599            runner.state = serde_json::from_value(legacy).unwrap();
15600            assert_eq!(runner.state.sources, refs::Sources::default());
15601            runner.state.save().unwrap();
15602            runner.state = RunState::load(&runner.state.id).unwrap();
15603            runner.prep().await.unwrap();
15604            for candidate in &runner.state.candidates {
15605                assert_eq!(
15606                    git::rev_parse(&candidate.worktree, "HEAD").await.unwrap(),
15607                    base
15608                );
15609                assert!(!candidate.worktree.join("unwanted").exists());
15610            }
15611        }
15612    }
15613
15614    #[tokio::test]
15615    async fn preparation_retries_after_explicit_release_and_rebuilds_partial_candidates() {
15616        ask_test_home();
15617        let tmp = tempfile::tempdir().unwrap();
15618        let repo = tmp.path().join("repo");
15619        std::fs::create_dir_all(&repo).unwrap();
15620        init_repo(&repo);
15621        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
15622        git::git(&repo, &["checkout", "-b", "seed"]).await.unwrap();
15623        std::fs::write(repo.join("referenced"), "required commit").unwrap();
15624        git::git(&repo, &["add", "."]).await.unwrap();
15625        git::git(&repo, &["commit", "-m", "required seed"])
15626            .await
15627            .unwrap();
15628        let seed = git::rev_parse(&repo, "HEAD").await.unwrap();
15629        git::git(&repo, &["checkout", "main"]).await.unwrap();
15630        let mut runner = runner_at(RunStatus::Prep);
15631        runner.state.repo = repo.clone();
15632        runner.state.base_commit = base.clone();
15633        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15634        runner.state.config.graph.advise = false;
15635        runner.state.config.blind.commit_msg_hook = false;
15636        runner.state.sources =
15637            refs::Sources::resolve(&repo, "origin", None, std::slice::from_ref(&seed))
15638                .await
15639                .unwrap();
15640        runner.roles.implementers = vec![spec("one"), spec("two")];
15641        runner.roles.judges = vec![spec("judge")];
15642        let label = blind::assign_labels(2, runner.state.seed)[0];
15643        let branch = runner.state.branch_for(label);
15644        let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15645        git::worktree_add_branch(&repo, &worktree, &branch, &base)
15646            .await
15647            .unwrap();
15648        runner.state.candidates.push(Candidate {
15649            index: 0,
15650            label,
15651            agent: "one".into(),
15652            branch,
15653            worktree,
15654            summary: String::new(),
15655            stat: String::new(),
15656            files: 0,
15657            commits: 0,
15658            empty: false,
15659            failed: None,
15660            verified_noop: None,
15661            duration_ms: 0,
15662            folded: false,
15663        });
15664        runner.state.save().unwrap();
15665        // Reload the checkpoint immediately before its first cherry-pick.
15666        runner.state = RunState::load(&runner.state.id).unwrap();
15667        assert_eq!(runner.state.prep_complete, Some(false));
15668        runner.prep().await.unwrap();
15669        assert_eq!(runner.state.prep_complete, Some(true));
15670        assert_eq!(runner.state.candidates.len(), 2);
15671        for candidate in &runner.state.candidates {
15672            assert_eq!(
15673                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15674                git::tree_of(&repo, &seed).await.unwrap()
15675            );
15676            assert_eq!(
15677                std::fs::read_to_string(candidate.worktree.join("referenced")).unwrap(),
15678                "required commit"
15679            );
15680        }
15681        assert!(runner.state.worktree_root().join("judge-1").exists());
15682        let heads: Vec<String> = candidate_heads(&runner.state.candidates).await;
15683        runner.prep().await.unwrap();
15684        assert_eq!(candidate_heads(&runner.state.candidates).await, heads);
15685
15686        runner.state.prep_complete = Some(false);
15687        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
15688            run: Some(format!(
15689                "test -f '{}'",
15690                tmp.path().join("dependency-ready").display()
15691            )),
15692            ..Default::default()
15693        }];
15694        assert!(runner.prep().await.is_err());
15695        assert!(runner.state.prep_failure.is_some());
15696        std::fs::write(tmp.path().join("dependency-ready"), "available").unwrap();
15697        runner.state = RunState::load(&runner.state.id).unwrap();
15698        assert!(
15699            runner.prep().await.is_err(),
15700            "correcting the cause alone must not auto-retry"
15701        );
15702        let failed_at = runner.state.events.last().unwrap().at;
15703        assert!(
15704            runner.retry_preparation(failed_at).is_err(),
15705            "old approvals cannot unlock the hold"
15706        );
15707        let mut task = crate::queue::Task::new(
15708            "retry".into(),
15709            "retry setup".into(),
15710            repo.clone(),
15711            crate::queue::Source::Human,
15712        );
15713        task.start(runner.state.id.clone());
15714        task.hold_machine(Some("setup failed".into()));
15715        task.release_operator("cli", None);
15716        runner
15717            .retry_preparation(task.resumption.as_ref().unwrap().at)
15718            .unwrap();
15719        runner.prep().await.unwrap();
15720        assert!(runner.state.prep_failure.is_none());
15721        assert_eq!(runner.state.prep_complete, Some(true));
15722        assert_eq!(runner.state.candidates.len(), 2);
15723        assert!(runner.state.worktree_root().join("judge-1").exists());
15724        for candidate in &runner.state.candidates {
15725            assert_eq!(
15726                git::tree_of(&candidate.worktree, "HEAD").await.unwrap(),
15727                git::tree_of(&repo, &seed).await.unwrap()
15728            );
15729        }
15730    }
15731
15732    async fn candidate_heads(candidates: &[Candidate]) -> Vec<String> {
15733        let mut heads = Vec::new();
15734        for candidate in candidates {
15735            heads.push(git::rev_parse(&candidate.worktree, "HEAD").await.unwrap());
15736        }
15737        heads
15738    }
15739
15740    async fn preservation_resume_fixture() -> (tempfile::TempDir, Runner) {
15741        ask_test_home();
15742        let tmp = tempfile::tempdir().unwrap();
15743        let repo = tmp.path().join("repo");
15744        std::fs::create_dir(&repo).unwrap();
15745        init_repo(&repo);
15746        let mut runner = runner_at(RunStatus::Reviewing);
15747        runner.state.repo = repo.clone();
15748        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
15749        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
15750        for (index, label) in ['A', 'B'].into_iter().enumerate() {
15751            let branch = format!("candidate-{label}");
15752            let worktree = runner.state.worktree_root().join(format!("cand-{label}"));
15753            git::worktree_add_branch(&repo, &worktree, &branch, "main")
15754                .await
15755                .unwrap();
15756            if label == 'B' {
15757                std::fs::write(worktree.join("change"), "winner").unwrap();
15758                git::rescue_commit(&worktree, "candidate change")
15759                    .await
15760                    .unwrap();
15761            }
15762            runner.state.candidates.push(Candidate {
15763                index,
15764                label,
15765                agent: "alpha".into(),
15766                branch,
15767                worktree,
15768                summary: "completed reply".into(),
15769                stat: String::new(),
15770                files: usize::from(label == 'B'),
15771                commits: usize::from(label == 'B'),
15772                empty: label == 'A',
15773                failed: None,
15774                verified_noop: None,
15775                duration_ms: 1,
15776                folded: false,
15777            });
15778        }
15779        runner.state.judge_skipped = true;
15780        runner.state.tally = Some(Tally {
15781            first_choice: BTreeMap::new(),
15782            borda: BTreeMap::new(),
15783            winner: 'B',
15784            rankings: 0,
15785            unanimous_initial: false,
15786            deliberated: false,
15787            changed_votes: 0,
15788            unanimous_final: false,
15789            tie_break: None,
15790            judges: 0,
15791            present: 0,
15792            quorum: 0,
15793            met_quorum: true,
15794            uncontested: Some("only B produced a change".into()),
15795        });
15796        runner.state.save().unwrap();
15797        (tmp, runner)
15798    }
15799
15800    #[tokio::test]
15801    #[cfg(unix)]
15802    async fn disk_interrupted_empty_implementation_retries_after_reload_and_failed_checkpoint() {
15803        for (solo, failed_checkpoint) in
15804            [(true, false), (false, false), (true, true), (false, true)]
15805        {
15806            let (_tmp, mut runner) = preservation_resume_fixture().await;
15807            if solo {
15808                runner.state.candidates.retain(|c| c.label == 'A');
15809            }
15810            runner.state.candidates[0].empty = false;
15811            runner.state.candidates[0].summary.clear();
15812            runner.state.tally = None;
15813            runner.state.judge_skipped = false;
15814            runner.state.status = RunStatus::Implementing;
15815            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15816            let mut implementer = spec("alpha");
15817            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15818            runner.roles.implementers = vec![implementer];
15819            let patch = runner.state.dir().join("artifacts/cand-A.patch");
15820            if failed_checkpoint {
15821                std::fs::create_dir_all(&patch).unwrap();
15822            }
15823            runner.state.save().unwrap();
15824            assert_eq!(runner.implement().await.is_err(), failed_checkpoint);
15825            drop(pressure);
15826            runner.state = RunState::load(&runner.state.id).unwrap();
15827            assert_eq!(runner.state.status, RunStatus::Blocked);
15828            assert!(runner.state.interrupted_implementations.contains(&'A'));
15829            assert_eq!(runner.state.candidates[0].commits, 0);
15830            assert!(!runner.state.candidates[0].empty);
15831            assert!(runner.state.candidates[0].failed.is_none());
15832            if failed_checkpoint {
15833                assert!(runner.state.candidate_preservation.contains_key(&'A'));
15834                std::fs::remove_dir(patch).unwrap();
15835            }
15836            runner.state.resource_block = None;
15837            runner.state.config.disk.min_free_bytes = 0;
15838            runner.roles.implementers[0].command = vec![
15839                "sh".into(),
15840                "-c".into(),
15841                "printf 'retried empty interrupted seat\\n' > retry-source; echo completed".into(),
15842            ];
15843            runner.implement().await.unwrap();
15844            assert!(runner.state.interrupted_implementations.is_empty());
15845            assert!(runner.state.candidate_preservation.is_empty());
15846            assert_eq!(runner.state.candidates[0].commits, 1);
15847            assert!(!runner.state.candidates[0].empty);
15848            assert!(
15849                std::fs::read_to_string(runner.state.candidates[0].worktree.join("retry-source"))
15850                    .unwrap()
15851                    .contains("retried empty interrupted seat")
15852            );
15853            if !solo {
15854                assert_eq!(runner.state.candidates[1].commits, 1);
15855            }
15856        }
15857    }
15858
15859    #[tokio::test]
15860    #[cfg(unix)]
15861    async fn disk_interrupted_implementation_preserves_edits_and_commits_before_resource_hold() {
15862        for (committed, block_rescue) in [(false, false), (false, true), (true, false)] {
15863            let (_tmp, mut runner) = preservation_resume_fixture().await;
15864            runner.state.candidates.retain(|c| c.label == 'A');
15865            runner.state.candidates[0].empty = false;
15866            runner.state.candidates[0].summary.clear();
15867            runner.state.tally = None;
15868            runner.state.judge_skipped = false;
15869            runner.state.status = RunStatus::Implementing;
15870            // Force only the real post-spawn watchdog, without failing the
15871            // separate admission check. Existing edits predate its measurement.
15872            let pressure = agent::test_watchdog::set(&runner.state.id, u64::MAX);
15873            let wt = runner.state.candidates[0].worktree.clone();
15874            std::fs::write(
15875                wt.join("interrupted-source"),
15876                "retain implementation work\n",
15877            )
15878            .unwrap();
15879            if committed {
15880                git::rescue_commit(&wt, "implementation checkpoint")
15881                    .await
15882                    .unwrap();
15883            }
15884            let lock = git::git(
15885                &wt,
15886                &[
15887                    "rev-parse",
15888                    "--path-format=absolute",
15889                    "--git-path",
15890                    "index.lock",
15891                ],
15892            )
15893            .await
15894            .unwrap();
15895            if block_rescue {
15896                std::fs::write(&lock, "fixture-owned lock").unwrap();
15897            }
15898            let mut implementer = spec("alpha");
15899            implementer.command = vec!["sh".into(), "-c".into(), "exec sleep 30".into()];
15900            runner.roles.implementers = vec![implementer];
15901            runner.state.save().unwrap();
15902            let result = runner.implement().await;
15903            drop(pressure);
15904            assert_eq!(result.is_err(), block_rescue);
15905            assert_eq!(
15906                runner.state.resource_block.as_ref().unwrap().kind,
15907                "disk pressure"
15908            );
15909            assert!(
15910                runner
15911                    .state
15912                    .events
15913                    .iter()
15914                    .any(|e| e.message.contains("agent execution interrupted"))
15915            );
15916            assert!(
15917                runner
15918                    .state
15919                    .dir()
15920                    .join("artifacts/candidate-A-recovery/staged.patch")
15921                    .exists()
15922            );
15923            if block_rescue {
15924                let saved = RunState::load(&runner.state.id).unwrap();
15925                assert!(saved.candidate_preservation.contains_key(&'A'));
15926                assert!(!saved.candidates[0].empty);
15927                assert_eq!(
15928                    std::fs::read_to_string(wt.join("interrupted-source")).unwrap(),
15929                    "retain implementation work\n"
15930                );
15931                std::fs::remove_file(lock).unwrap();
15932                runner.state = saved;
15933            } else {
15934                assert_eq!(runner.state.status, RunStatus::Blocked);
15935                assert_eq!(runner.state.candidates[0].commits, 1);
15936            }
15937            // Recovery must save the candidate before deciding whether another
15938            // implementation is needed. No implementer is available on retry.
15939            runner.roles.implementers.clear();
15940            runner.state.config.disk.min_free_bytes = 0;
15941            runner.state.resource_block = None;
15942            runner.implement().await.unwrap();
15943            assert!(runner.state.candidate_preservation.is_empty());
15944            assert_eq!(runner.state.candidates[0].commits, 1);
15945            assert!(!runner.state.candidates[0].empty);
15946            assert!(git::is_clean(&wt).await.unwrap());
15947            assert!(
15948                crate::run::read_artifact(&runner.state, "cand-A.patch")
15949                    .unwrap()
15950                    .contains("retain implementation work")
15951            );
15952        }
15953    }
15954
15955    #[tokio::test]
15956    async fn candidate_preservation_does_not_reflag_confirmed_or_folded_empty_candidates() {
15957        let (_tmp, mut runner) = preservation_resume_fixture().await;
15958        // Schema 15 already confirmed the unfolded no-change result.
15959        runner.implement().await.unwrap();
15960        assert!(runner.state.candidate_preservation.is_empty());
15961        assert!(
15962            !runner
15963                .state
15964                .dir()
15965                .join("artifacts/candidate-A-recovery")
15966                .exists()
15967        );
15968        runner.fold_losers().await.unwrap();
15969        assert!(!runner.state.candidates[0].worktree.exists());
15970        for schema in [14, crate::run::SCHEMA] {
15971            runner.state.schema = schema;
15972            runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
15973            runner.state.save().unwrap();
15974            runner.state = RunState::load(&runner.state.id).unwrap();
15975            runner.implement().await.unwrap();
15976            assert!(runner.state.candidate_preservation.is_empty());
15977            assert!(runner.state.candidates[0].folded);
15978            assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
15979        }
15980    }
15981
15982    #[tokio::test]
15983    async fn infrastructure_failure_report_retains_eligible_candidate_edits() {
15984        let (_tmp, mut runner) = preservation_resume_fixture().await;
15985        runner.state.candidates.retain(|c| c.label == 'A');
15986        runner.state.tally = None;
15987        runner.state.judge_skipped = false;
15988        let candidate = &mut runner.state.candidates[0];
15989        candidate.summary = "Reproduction bind denied with EPERM; source repair complete, deployed browser validation remains parent-owned.".into();
15990        candidate.failed = Some("deployment validation unavailable".into());
15991        std::fs::write(
15992            candidate.worktree.join("repair"),
15993            "future launcher policy\n",
15994        )
15995        .unwrap();
15996        runner
15997            .state
15998            .candidate_preservation
15999            .insert('A', "awaiting persistence".into());
16000        runner.persist_candidates().await.unwrap();
16001        let candidate = &runner.state.candidates[0];
16002        assert!(!candidate.empty);
16003        assert_eq!(candidate.commits, 1);
16004        assert_eq!(candidate.files, 1);
16005        assert!(candidate.failed.is_none());
16006        assert!(candidate.summary.contains("EPERM"));
16007        assert!(
16008            crate::run::read_artifact(&runner.state, "cand-A.patch")
16009                .unwrap()
16010                .contains("future launcher policy")
16011        );
16012        assert!(runner.state.candidate_preservation.is_empty());
16013    }
16014
16015    #[tokio::test]
16016    async fn candidate_preservation_legacy_recovery_blocks_stale_verdict_on_every_resume() {
16017        let (_tmp, mut runner) = preservation_resume_fixture().await;
16018        let wt = runner.state.candidates[0].worktree.clone();
16019        std::fs::write(wt.join("recovered"), "owned edits").unwrap();
16020        runner.state.schema = 14;
16021        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16022        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16023        let tally = serde_json::to_value(&runner.state.tally).unwrap();
16024        for _ in 0..2 {
16025            assert!(runner.implement().await.is_err());
16026            assert_eq!(runner.state.status, RunStatus::Blocked);
16027            assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16028            assert!(
16029                runner.state.candidates[0].empty,
16030                "old verdict must remain unusable"
16031            );
16032            assert_eq!(serde_json::to_value(&runner.state.tally).unwrap(), tally);
16033            assert!(runner.state.judge_skipped);
16034            assert!(runner.state.judgements.is_empty() && runner.state.votes.is_empty());
16035            assert!(
16036                crate::run::read_artifact(&runner.state, "cand-A.patch")
16037                    .unwrap()
16038                    .contains("owned edits")
16039            );
16040            assert!(wt.exists());
16041            assert_eq!(
16042                git::commits_ahead(&wt, &runner.state.base_commit, "HEAD")
16043                    .await
16044                    .unwrap(),
16045                1
16046            );
16047            runner.state = RunState::load(&runner.state.id).unwrap();
16048        }
16049    }
16050
16051    #[tokio::test]
16052    async fn candidate_preservation_legacy_failed_seat_keeps_edits_before_judging() {
16053        let (_tmp, mut runner) = preservation_resume_fixture().await;
16054        let wt = runner.state.candidates[0].worktree.clone();
16055        runner.state.candidates[0].failed = Some("CLI failed before commit".into());
16056        std::fs::write(wt.join("recovered"), "failed seat edits").unwrap();
16057        let index_lock = PathBuf::from(
16058            git::git(
16059                &wt,
16060                &[
16061                    "rev-parse",
16062                    "--path-format=absolute",
16063                    "--git-path",
16064                    "index.lock",
16065                ],
16066            )
16067            .await
16068            .unwrap(),
16069        );
16070        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16071        runner.state.schema = 14;
16072        runner.state = crate::run::migrate_schema(runner.state.clone()).unwrap();
16073        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16074        assert!(runner.implement().await.is_err());
16075        assert!(runner.after_implement().is_err());
16076        assert!(runner.fold_losers().await.is_err());
16077        assert_eq!(
16078            std::fs::read_to_string(wt.join("recovered")).unwrap(),
16079            "failed seat edits"
16080        );
16081        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16082        runner.state = RunState::load(&runner.state.id).unwrap();
16083        std::fs::remove_file(index_lock).unwrap();
16084        // Recovery finds changes that the old uncontested result never judged.
16085        assert!(runner.implement().await.is_err());
16086        assert!(runner.state.candidate_preservation[&'A'].contains("supported recovery"));
16087        assert!(runner.state.candidates[0].empty);
16088        assert_eq!(runner.state.tally.as_ref().unwrap().winner, 'B');
16089        assert!(
16090            crate::run::read_artifact(&runner.state, "cand-A.patch")
16091                .unwrap()
16092                .contains("failed seat edits")
16093        );
16094        assert!(wt.exists());
16095    }
16096
16097    #[tokio::test]
16098    async fn candidate_preservation_loser_cleanup_resumes_after_branch_and_judge_failures() {
16099        let (_tmp, mut runner) = preservation_resume_fixture().await;
16100        let repo = runner.state.repo.clone();
16101        let common = git::git(
16102            &repo,
16103            &["rev-parse", "--path-format=absolute", "--git-common-dir"],
16104        )
16105        .await
16106        .unwrap();
16107        let branch_lock = PathBuf::from(common).join("refs/heads/candidate-A.lock");
16108        std::fs::write(&branch_lock, "fixture-owned lock").unwrap();
16109        assert!(runner.fold_losers().await.is_err());
16110        assert!(!runner.state.candidates[0].worktree.exists());
16111        assert!(!runner.state.candidates[0].folded);
16112        assert!(git::branch_exists(&repo, "candidate-A").await.unwrap());
16113        runner.state = RunState::load(&runner.state.id).unwrap();
16114        std::fs::remove_file(branch_lock).unwrap();
16115        let judge = runner.state.worktree_root().join("judge-1");
16116        git::worktree_add_detached(&repo, &judge, "main")
16117            .await
16118            .unwrap();
16119        std::fs::write(judge.join("uncommitted"), "retain").unwrap();
16120        runner.roles.judges = vec![spec("judge")];
16121        assert!(runner.fold_losers().await.is_err());
16122        runner.state = RunState::load(&runner.state.id).unwrap();
16123        assert!(
16124            runner.state.candidates[0].folded,
16125            "candidate progress saved before judge failure"
16126        );
16127        assert!(!git::branch_exists(&repo, "candidate-A").await.unwrap());
16128        assert!(judge.join("uncommitted").exists());
16129        std::fs::remove_file(judge.join("uncommitted")).unwrap();
16130        runner.fold_losers().await.unwrap();
16131        assert!(!judge.exists());
16132        runner.fold_losers().await.unwrap();
16133    }
16134
16135    #[tokio::test]
16136    async fn candidate_preservation_selected_implementer_resumes_through_execute_graph() {
16137        let (_tmp, mut runner) = preservation_resume_fixture().await;
16138        runner.state.candidates.retain(|c| c.label == 'B');
16139        runner.state.tally = None;
16140        runner.state.judge_skipped = false;
16141        runner.state.prep_complete = Some(true);
16142        runner.state.config.graph.advise = false;
16143        runner.state.config.graph.selected_implementer = Some("alpha".into());
16144        let wt = runner.state.candidates[0].worktree.clone();
16145        std::fs::write(wt.join("leftover"), "completed implementation").unwrap();
16146        runner
16147            .state
16148            .candidate_preservation
16149            .insert('B', "awaiting persistence".into());
16150        let index_lock = git::git(
16151            &wt,
16152            &[
16153                "rev-parse",
16154                "--path-format=absolute",
16155                "--git-path",
16156                "index.lock",
16157            ],
16158        )
16159        .await
16160        .unwrap();
16161        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16162        runner.state.save().unwrap();
16163        assert!(runner.execute_graph().await.is_err());
16164        assert_eq!(runner.state.status, RunStatus::Blocked);
16165        assert!(runner.state.candidate_preservation.contains_key(&'B'));
16166        std::fs::remove_file(index_lock).unwrap();
16167        // Park at the next graph boundary: this tests the real selected-seat
16168        // hold guard without dispatching reviewers or publishing anything.
16169        for _ in 0..2 {
16170            runner.state = RunState::load(&runner.state.id).unwrap();
16171            runner.pause.park();
16172            runner.execute_graph().await.unwrap();
16173            assert_eq!(runner.state.status, RunStatus::Implementing);
16174            assert!(runner.state.candidate_preservation.is_empty());
16175            assert_eq!(runner.state.candidates[0].commits, 2);
16176            assert!(runner.state.tally.is_none());
16177        }
16178        runner.implement().await.unwrap();
16179        assert_eq!(runner.state.status, RunStatus::Judging);
16180        // A genuine selected-implementer hold must still require release.
16181        runner.state.status = RunStatus::Blocked;
16182        runner.state.candidates[0].failed = Some("implementation failed".into());
16183        runner.state.save().unwrap();
16184        let error = runner.execute_graph().await.unwrap_err();
16185        assert!(
16186            error
16187                .to_string()
16188                .contains("selected implementation is held")
16189        );
16190    }
16191
16192    #[tokio::test]
16193    async fn candidate_preservation_cleanup_unregisters_a_vanished_worktree() {
16194        let (_tmp, mut runner) = preservation_resume_fixture().await;
16195        let repo = runner.state.repo.clone();
16196        let candidate = runner.state.candidates[0].clone();
16197        // Remove only the disposable fixture directory, leaving Git's linked
16198        // worktree registration and branch intact, as in a vanished checkout.
16199        let registered = git::worktree_holding(&repo, &candidate.branch)
16200            .await
16201            .unwrap()
16202            .unwrap();
16203        assert_eq!(
16204            registered.canonicalize().unwrap(),
16205            candidate.worktree.canonicalize().unwrap()
16206        );
16207        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16208        assert_eq!(
16209            git::worktree_holding(&repo, &candidate.branch)
16210                .await
16211                .unwrap(),
16212            Some(registered)
16213        );
16214        assert!(
16215            !git::branch_delete(&repo, &candidate.branch).await.unwrap(),
16216            "the stale registration initially keeps the branch checked out"
16217        );
16218        runner.fold_losers().await.unwrap();
16219        assert!(runner.state.candidates[0].folded);
16220        assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16221        assert_eq!(
16222            git::worktree_holding(&repo, &candidate.branch)
16223                .await
16224                .unwrap(),
16225            None
16226        );
16227        runner.state = RunState::load(&runner.state.id).unwrap();
16228        runner.fold_losers().await.unwrap();
16229    }
16230
16231    #[tokio::test]
16232    async fn fold_run_unregisters_a_vanished_worktree_and_can_retry() {
16233        let (_tmp, mut runner) = preservation_resume_fixture().await;
16234        let repo = runner.state.repo.clone();
16235        let candidate = runner.state.candidates[0].clone();
16236        let winner = runner.state.candidates[1].clone();
16237        let registered = git::worktree_holding(&repo, &candidate.branch)
16238            .await
16239            .unwrap()
16240            .unwrap();
16241        assert_eq!(
16242            registered.canonicalize().unwrap(),
16243            candidate.worktree.canonicalize().unwrap()
16244        );
16245        std::fs::remove_dir_all(&candidate.worktree).unwrap();
16246        assert_eq!(
16247            git::worktree_holding(&repo, &candidate.branch)
16248                .await
16249                .unwrap(),
16250            Some(registered)
16251        );
16252        assert!(!git::branch_delete(&repo, &candidate.branch).await.unwrap());
16253
16254        for _ in 0..2 {
16255            fold_run(&mut runner.state, false, &crate::run::home())
16256                .await
16257                .unwrap();
16258            runner.state = RunState::load(&runner.state.id).unwrap();
16259            assert!(runner.state.candidates[0].folded);
16260            assert!(!git::branch_exists(&repo, &candidate.branch).await.unwrap());
16261            assert_eq!(
16262                git::worktree_holding(&repo, &candidate.branch)
16263                    .await
16264                    .unwrap(),
16265                None
16266            );
16267            assert!(!runner.state.candidates[1].folded);
16268            assert!(winner.worktree.exists());
16269            assert!(git::branch_exists(&repo, &winner.branch).await.unwrap());
16270        }
16271    }
16272
16273    #[tokio::test]
16274    async fn candidate_preservation_retries_after_upgrade_without_rerunning_seats() {
16275        ask_test_home();
16276        let tmp = tempfile::tempdir().unwrap();
16277        let repo = tmp.path().join("repo");
16278        std::fs::create_dir(&repo).unwrap();
16279        init_repo(&repo);
16280        let mut runner = runner_at(RunStatus::Implementing);
16281        runner.state.repo = repo.clone();
16282        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16283        runner.state.config.graph.worktree_root = Some(tmp.path().join("wt"));
16284        for (index, label) in ['A', 'B'].into_iter().enumerate() {
16285            let branch = format!("candidate-{label}");
16286            let wt = runner.state.worktree_root().join(format!("cand-{label}"));
16287            git::worktree_add_branch(&repo, &wt, &branch, "main")
16288                .await
16289                .unwrap();
16290            std::fs::write(wt.join("README.md"), "tracked edits\n").unwrap();
16291            git::git(&wt, &["add", "README.md"]).await.unwrap();
16292            std::fs::write(wt.join("README.md"), "later tracked edits\n").unwrap();
16293            std::fs::write(wt.join("new-file"), "untracked edits\n").unwrap();
16294            runner.state.candidates.push(Candidate {
16295                index,
16296                label,
16297                agent: "alpha".to_owned(),
16298                branch,
16299                worktree: wt,
16300                summary: "completed implementation".to_owned(),
16301                stat: String::new(),
16302                files: 0,
16303                commits: 0,
16304                empty: false,
16305                failed: None,
16306                verified_noop: None,
16307                duration_ms: 42,
16308                folded: false,
16309            });
16310            runner
16311                .state
16312                .candidate_preservation
16313                .insert(label, "awaiting persistence".to_owned());
16314        }
16315        let wt = runner.state.candidates[0].worktree.clone();
16316        let index_lock = git::git(
16317            &wt,
16318            &[
16319                "rev-parse",
16320                "--path-format=absolute",
16321                "--git-path",
16322                "index.lock",
16323            ],
16324        )
16325        .await
16326        .unwrap();
16327        std::fs::write(&index_lock, "fixture-owned lock").unwrap();
16328        runner.state.save().unwrap();
16329        assert!(runner.persist_candidates().await.is_err());
16330        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16331        assert!(!runner.state.candidate_preservation.contains_key(&'B'));
16332        assert_eq!(
16333            runner.state.candidates[1].commits, 1,
16334            "other results are still collected"
16335        );
16336        assert!(
16337            !runner.state.candidates[0].empty,
16338            "failed rescue is not no change"
16339        );
16340        assert_eq!(
16341            runner.state.candidates[0].summary,
16342            "completed implementation"
16343        );
16344        assert!(runner.after_implement().is_err());
16345        let home = tmp.path().join("fold-home");
16346        assert!(fold_run(&mut runner.state, true, &home).await.is_err());
16347        assert!(wt.join("new-file").exists());
16348        let recovery = runner.state.dir().join("artifacts/candidate-A-recovery");
16349        assert!(recovery.join("staged.patch").exists());
16350        assert!(recovery.join("unstaged.patch").exists());
16351        assert_eq!(
16352            std::fs::read_to_string(recovery.join("untracked/new-file")).unwrap(),
16353            "untracked edits\n"
16354        );
16355        runner.state.begin_pass();
16356        runner.state.end_pass_parked(ParkCause::Upgrade {
16357            from: "1.0".into(),
16358            to: Some("1.1".into()),
16359        });
16360        runner.state.save().unwrap();
16361        runner.state = RunState::load(&runner.state.id).unwrap();
16362        assert!(matches!(
16363            runner.state.passes.last().unwrap().end,
16364            Some(crate::run::PassEnd::Parked {
16365                cause: ParkCause::Upgrade { .. }
16366            })
16367        ));
16368        std::fs::remove_file(index_lock).unwrap();
16369        // No implementer is configured: a persistence retry cannot dispatch a seat.
16370        runner.implement().await.unwrap();
16371        assert!(runner.state.candidate_preservation.is_empty());
16372        assert!(
16373            runner
16374                .state
16375                .candidates
16376                .iter()
16377                .all(|c| c.commits == 1 && !c.empty)
16378        );
16379        assert!(git::is_clean(&wt).await.unwrap());
16380        assert!(
16381            crate::run::read_artifact(&runner.state, "cand-A.patch")
16382                .unwrap()
16383                .contains("later tracked edits")
16384        );
16385    }
16386
16387    #[tokio::test]
16388    async fn candidate_preservation_git_read_and_cleanup_fail_closed() {
16389        ask_test_home();
16390        let tmp = tempfile::tempdir().unwrap();
16391        let repo = tmp.path().join("repo");
16392        std::fs::create_dir(&repo).unwrap();
16393        init_repo(&repo);
16394        let wt = tmp.path().join("candidate");
16395        git::worktree_add_branch(&repo, &wt, "candidate", "main")
16396            .await
16397            .unwrap();
16398        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16399        assert!(remove_preserved_worktree(&repo, &wt).await.is_err());
16400        assert!(wt.join("uncommitted").exists());
16401        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16402        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16403        let mut runner = runner_at(RunStatus::Implementing);
16404        runner.state.repo = repo.clone();
16405        runner.state.base_commit = "missing-base".to_owned();
16406        runner.state.candidates.push(Candidate {
16407            index: 0,
16408            label: 'A',
16409            agent: "alpha".into(),
16410            branch: "candidate".into(),
16411            worktree: wt.clone(),
16412            summary: "completed".into(),
16413            stat: String::new(),
16414            files: 0,
16415            commits: 0,
16416            empty: false,
16417            failed: None,
16418            verified_noop: None,
16419            duration_ms: 1,
16420            folded: false,
16421        });
16422        std::fs::write(wt.join("uncommitted"), "keep").unwrap();
16423        assert!(
16424            fold_run(&mut runner.state, true, &tmp.path().join("home"))
16425                .await
16426                .is_err()
16427        );
16428        assert!(!runner.state.candidates[0].folded);
16429        assert!(git::branch_exists(&repo, "candidate").await.unwrap());
16430        std::fs::remove_file(wt.join("uncommitted")).unwrap();
16431        runner
16432            .state
16433            .candidate_preservation
16434            .insert('A', "pending".into());
16435        assert!(runner.persist_candidates().await.is_err());
16436        assert!(!runner.state.candidates[0].empty);
16437        assert!(!runner.state.candidates[0].folded);
16438        assert!(wt.exists());
16439        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
16440        let patch_path = runner.state.dir().join("artifacts/cand-A.patch");
16441        std::fs::create_dir_all(&patch_path).unwrap();
16442        assert!(
16443            runner.persist_candidates().await.is_err(),
16444            "artifact failure stays pending"
16445        );
16446        assert!(runner.state.candidate_preservation.contains_key(&'A'));
16447        assert!(!runner.state.candidates[0].empty);
16448        std::fs::remove_dir(patch_path).unwrap();
16449        runner.persist_candidates().await.unwrap();
16450        assert!(
16451            runner.state.candidates[0].empty,
16452            "only successful inspection confirms no change"
16453        );
16454    }
16455
16456    fn init_repo(dir: &Path) {
16457        let run = |args: &[&str]| {
16458            let out = std::process::Command::new("git")
16459                .args(args)
16460                .current_dir(dir)
16461                .quiet()
16462                .output()
16463                .expect("spawn git");
16464            assert!(
16465                out.status.success(),
16466                "git {args:?} failed: {}",
16467                String::from_utf8_lossy(&out.stderr)
16468            );
16469        };
16470        run(&["init", "-b", "main"]);
16471        run(&["config", "user.name", "magi test"]);
16472        run(&["config", "user.email", "magi@example.com"]);
16473        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
16474        run(&["add", "-A"]);
16475        run(&["commit", "-m", "init"]);
16476    }
16477
16478    // `settle_questions` is what closes the ghost the phone showed: a run's
16479    // seat asked something, the run then ended, and nothing was left to
16480    // abandon the question it left `open`. `HOME` is a process-wide
16481    // `OnceLock` (see `run::set_home`'s doc), so this only wins the race the
16482    // first time it runs in the binary — every test below still reaches the
16483    // same directory whichever call won, and each gets its own run id from
16484    // `RunState::new`, so they never collide there.
16485    fn ask_test_home() {
16486        crate::run::pin_test_home();
16487    }
16488
16489    #[test]
16490    fn startup_terminal_save_failure_preserves_original_error_and_reservation() {
16491        ask_test_home();
16492        let mut runner = runner_at(RunStatus::Prep);
16493        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16494        // A blocked temporary output path makes the terminal save fail while
16495        // preserving the already published reservation.
16496        std::fs::create_dir(runner.state.dir().join("run.json.tmp")).unwrap();
16497        let error = guard
16498            .finish(
16499                &mut runner.state,
16500                Err(
16501                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "original refusal")
16502                        .into(),
16503                ),
16504            )
16505            .unwrap_err();
16506        assert_eq!(
16507            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16508            std::io::ErrorKind::PermissionDenied
16509        );
16510        assert!(error.downcast_ref::<crate::run::StartupFailure>().is_some());
16511        let message = format!("{error:#}");
16512        assert!(message.contains("original refusal"));
16513        assert!(!message.contains("could not save terminal"));
16514        assert_eq!(
16515            error.chain().count(),
16516            2,
16517            "only startup attribution wraps the original error"
16518        );
16519        let saved = RunState::load(&runner.state.id).unwrap();
16520        assert_eq!(saved.status, RunStatus::Blocked);
16521        assert!(!saved.startup.unwrap().complete);
16522    }
16523
16524    #[test]
16525    fn reserved_startup_interruption_is_blocked_and_not_resumable() {
16526        ask_test_home();
16527        let mut runner = runner_at(RunStatus::Prep);
16528        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16529        let id = runner.state.id.clone();
16530        drop(guard);
16531        let saved = RunState::load(&id).unwrap();
16532        assert_eq!(saved.status, RunStatus::Blocked);
16533        assert!(!saved.startup.as_ref().unwrap().complete);
16534        assert!(
16535            saved
16536                .startup
16537                .unwrap()
16538                .failure
16539                .unwrap()
16540                .contains("interrupted")
16541        );
16542        assert!(Runner::resume(&id).is_err());
16543    }
16544
16545    #[test]
16546    fn startup_failure_keeps_typed_cause_and_separate_task_history() {
16547        ask_test_home();
16548        let mut runner = runner_at(RunStatus::Prep);
16549        let guard = StartupGuard::reserve(&mut runner.state, "review", Some("feature")).unwrap();
16550        let error = guard
16551            .finish(
16552                &mut runner.state,
16553                Err(
16554                    std::io::Error::new(std::io::ErrorKind::PermissionDenied, "refused startup")
16555                        .into(),
16556                ),
16557            )
16558            .unwrap_err();
16559        assert_eq!(
16560            error.downcast_ref::<std::io::Error>().unwrap().kind(),
16561            std::io::ErrorKind::PermissionDenied
16562        );
16563        let failure = error.downcast_ref::<crate::run::StartupFailure>().unwrap();
16564        let mut task = crate::queue::Task::new(
16565            "review".into(),
16566            "criteria".into(),
16567            runner.state.repo.clone(),
16568            crate::queue::Source::Human,
16569        );
16570        task.record_startup_failure(failure.clone());
16571        task.record_startup_failure(failure.clone());
16572        assert_eq!(task.startup_failures.len(), 1);
16573        assert!(task.runs.is_empty());
16574        assert_eq!(task.attempts, 0);
16575    }
16576
16577    #[cfg(unix)]
16578    async fn exhausted_base_fixture() -> (tempfile::TempDir, Runner, String, String) {
16579        ask_test_home();
16580        let tmp = tempfile::tempdir().unwrap();
16581        let repo = tmp.path().join("repo");
16582        std::fs::create_dir(&repo).unwrap();
16583        init_repo(&repo);
16584        let base = git::rev_parse(&repo, "HEAD").await.unwrap();
16585        let origin = tmp.path().join("origin.git");
16586        git::git(&repo, &["init", "--bare", origin.to_str().unwrap()])
16587            .await
16588            .unwrap();
16589        git::git(
16590            &repo,
16591            &["remote", "add", "origin", origin.to_str().unwrap()],
16592        )
16593        .await
16594        .unwrap();
16595        git::git(&repo, &["checkout", "-b", "fix/retained"])
16596            .await
16597            .unwrap();
16598        std::fs::write(repo.join("retained"), "retained change").unwrap();
16599        git::git(&repo, &["add", "."]).await.unwrap();
16600        git::git(&repo, &["commit", "-m", "retained implementation"])
16601            .await
16602            .unwrap();
16603        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
16604        git::git(&repo, &["checkout", "main"]).await.unwrap();
16605        std::fs::write(repo.join("base-change"), "new base").unwrap();
16606        git::git(&repo, &["add", "."]).await.unwrap();
16607        git::git(&repo, &["commit", "-m", "base moved"])
16608            .await
16609            .unwrap();
16610        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16611        git::git(&repo, &["checkout", "fix/retained"])
16612            .await
16613            .unwrap();
16614        let mut runner = runner_at(RunStatus::Voting);
16615        runner.state.repo = repo.clone();
16616        runner.state.base_commit = base;
16617        runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
16618        runner.state.config.disk.min_free_bytes = 0;
16619        runner.state.config.graph.review_rounds = 1;
16620        runner.state.config.verify.e2e = vec!["true".into()];
16621        runner.state.config.verify.gate = vec!["true".into()];
16622        runner.state.candidates = vec![Candidate {
16623            index: 0,
16624            label: 'A',
16625            agent: "fixture".into(),
16626            branch: "fix/retained".into(),
16627            worktree: repo.clone(),
16628            summary: String::new(),
16629            stat: String::new(),
16630            files: 1,
16631            commits: 1,
16632            empty: false,
16633            failed: None,
16634            verified_noop: None,
16635            duration_ms: 0,
16636            folded: false,
16637        }];
16638        runner.state.tally = Some(Tally {
16639            first_choice: BTreeMap::new(),
16640            borda: BTreeMap::new(),
16641            winner: 'A',
16642            rankings: 0,
16643            unanimous_initial: true,
16644            deliberated: false,
16645            changed_votes: 0,
16646            unanimous_final: true,
16647            tie_break: None,
16648            judges: 0,
16649            present: 0,
16650            quorum: 0,
16651            met_quorum: true,
16652            uncontested: Some("review-only".into()),
16653        });
16654        let mut task = crate::queue::Task::new(
16655            "fixture".into(),
16656            "fixture".into(),
16657            repo,
16658            crate::queue::Source::Human,
16659        );
16660        task.link_run(&runner.state.id);
16661        crate::queue::Queue::open().put(&mut task).unwrap();
16662        runner.state.origin = Some(crate::run::Origin::queue(&task.id));
16663        runner.state.base_sync = Some(BaseSync {
16664            tip: String::new(),
16665            behind: 1,
16666            attempts: BASE_SYNC_ROUNDS,
16667            conflict: None,
16668            already_in: None,
16669        });
16670        runner.sync_to_base().await.unwrap();
16671        assert_eq!(runner.state.status, RunStatus::Blocked);
16672        (tmp, runner, head, task.id)
16673    }
16674
16675    #[cfg(unix)]
16676    #[tokio::test]
16677    async fn base_recovery_authorizes_its_worker_but_rejects_other_workers() {
16678        const CHILD: &str = "MAGI_BASE_RECOVERY_WORKER_TEST";
16679        if std::env::var_os(CHILD).is_none() {
16680            let home = tempfile::tempdir().unwrap();
16681            let output = std::process::Command::new(std::env::current_exe().unwrap())
16682                .args([
16683                    "--exact",
16684                    "graph::tests::base_recovery_authorizes_its_worker_but_rejects_other_workers",
16685                    "--nocapture",
16686                ])
16687                .env(CHILD, "1")
16688                .env("MAGI_HOME", home.path())
16689                .env("MAGI_WORKER_HOME", home.path())
16690                .env("MAGI_WORKER_ID", "abcdef")
16691                .output()
16692                .unwrap();
16693            assert!(
16694                output.status.success()
16695                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16696                "{}\n{}",
16697                String::from_utf8_lossy(&output.stdout),
16698                String::from_utf8_lossy(&output.stderr)
16699            );
16700            return;
16701        }
16702        crate::run::set_home(PathBuf::from(std::env::var_os("MAGI_HOME").unwrap()));
16703        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16704        let home = crate::run::home();
16705        let queue = crate::queue::Queue::open();
16706        let mut owner = queue.get(&task).unwrap();
16707        owner.release_operator("test-release", None);
16708        owner.start(runner.state.id.clone());
16709        queue.put(&mut owner).unwrap();
16710        runner.state.driver_exited = true;
16711        runner.state.task_instruction = Some(owner.instruction.clone());
16712        runner.state.save().unwrap();
16713        let _claim = queue.claim(&task).unwrap();
16714        let reserve = |id: &str| {
16715            let directory = home.join("workers/jobs").join(id);
16716            std::fs::create_dir_all(&directory).unwrap();
16717            let job = crate::worker::Job {
16718                id: id.into(),
16719                sequence: 1,
16720                kind: "task".into(),
16721                key: format!("task:{task}"),
16722                payload: serde_json::json!({"opts": crate::daemon::Opts::default(), "queue": queue.root(), "task": task}),
16723                generation: "fixture-generation".into(),
16724                version: None,
16725                status: crate::worker::Status::Running,
16726                result: None,
16727                pid: Some(std::process::id()),
16728                created_at: 0,
16729                finished_at: None,
16730            };
16731            std::fs::write(
16732                directory.join("job.json"),
16733                serde_json::to_vec(&job).unwrap(),
16734            )
16735            .unwrap();
16736            crate::worker::execution_lease_for_test(&directory.join("live.lock"))
16737        };
16738        let _own = reserve("abcdef");
16739        let other = reserve("fedcba");
16740        assert!(
16741            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16742                .await
16743                .is_err()
16744        );
16745        assert!(
16746            RunState::load(&runner.state.id)
16747                .unwrap()
16748                .base_sync_recovery
16749                .authorizations
16750                .is_empty()
16751        );
16752        drop(other);
16753        assert!(crate::daemon::is_working_on(
16754            &home,
16755            &runner.state.id,
16756            Timestamp::now()
16757        ));
16758        assert!(
16759            !crate::daemon::is_working_on_other(&home, &runner.state.id, Timestamp::now()).unwrap()
16760        );
16761        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
16762            .await
16763            .unwrap();
16764        assert!(
16765            RunState::load(&runner.state.id)
16766                .unwrap()
16767                .base_recovery_pending()
16768        );
16769    }
16770
16771    #[cfg(unix)]
16772    #[tokio::test]
16773    async fn base_recovery_authorizes_a_crashed_driver_with_a_free_matching_lease() {
16774        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16775        let home = crate::run::home();
16776        runner.state.driver_pid = Some(std::process::id());
16777        runner.state.driver_started_at = crate::proc::process_started_at(std::process::id());
16778        runner.state.driver_exited = false;
16779        runner.state.begin_pass();
16780        runner.state.save().unwrap();
16781        let mut driver = crate::driver::DriverGuard::acquire(&home, &runner.state).unwrap();
16782        driver.record(&runner.state).unwrap();
16783        let saved = std::fs::read(runner.state.dir().join("run.json")).unwrap();
16784        // Preserve the durable claim as if the process died before Drop saved.
16785        drop(driver);
16786        std::fs::write(runner.state.dir().join("run.json"), saved).unwrap();
16787        assert_eq!(
16788            runner.state.liveness_under(&home, false),
16789            crate::run::Liveness::Dead
16790        );
16791        let queue = crate::queue::Queue::open();
16792        let _claim = queue.claim(&task).unwrap();
16793        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16794            .await
16795            .unwrap();
16796        let disk = RunState::load(&runner.state.id).unwrap();
16797        assert!(!disk.driver_exited);
16798        assert!(disk.base_recovery_pending());
16799    }
16800
16801    #[cfg(unix)]
16802    #[tokio::test]
16803    async fn base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase() {
16804        const CHILD: &str = "MAGI_BASE_RECOVERY_CLOSURE_TEST";
16805        if std::env::var_os(CHILD).is_none() {
16806            use std::os::unix::fs::PermissionsExt;
16807            let tmp = tempfile::tempdir().unwrap();
16808            let gh = tmp.path().join("gh");
16809            std::fs::write(&gh, r#"#!/bin/sh
16810case "$1 $2" in
16811  'pr view') printf '{"headRefName":"fix/retained","headRefOid":"%s","baseRefName":"main","state":"OPEN","isCrossRepository":false}' "$(git rev-parse fix/retained)" ;;
16812  'pr close') if [ ! -f "$MAGI_BASE_CLOSURE_MARKER" ]; then touch "$MAGI_BASE_CLOSURE_MARKER"; echo 'transient closure failure' >&2; exit 1; fi ;;
16813  *) exit 2 ;;
16814esac
16815"#).unwrap();
16816            std::fs::set_permissions(&gh, std::fs::Permissions::from_mode(0o755)).unwrap();
16817            let mut paths = vec![tmp.path().to_path_buf()];
16818            paths.extend(std::env::split_paths(&std::env::var_os("PATH").unwrap()));
16819            let output = std::process::Command::new(std::env::current_exe().unwrap())
16820                .args(["--exact", "graph::tests::base_recovery_retries_integrated_pr_closure_without_authorizing_a_rebase", "--nocapture"])
16821                .env(CHILD, "1")
16822                .env("MAGI_BASE_CLOSURE_MARKER", tmp.path().join("closure-attempted"))
16823                .env("PATH", std::env::join_paths(paths).unwrap())
16824                .env_remove("MAGI_WORKER_HOME").env_remove("MAGI_WORKER_ID")
16825                .output().unwrap();
16826            assert!(
16827                output.status.success()
16828                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
16829                "{}\n{}",
16830                String::from_utf8_lossy(&output.stdout),
16831                String::from_utf8_lossy(&output.stderr)
16832            );
16833            return;
16834        }
16835        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16836        let repo = runner.state.repo.clone();
16837        git::git(&repo, &["checkout", "main"]).await.unwrap();
16838        git::git(&repo, &["cherry-pick", &head]).await.unwrap();
16839        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
16840        git::git(&repo, &["checkout", "fix/retained"])
16841            .await
16842            .unwrap();
16843        runner.state.pr = Some(crate::run::PrRecord {
16844            url: "https://example.test/project/pull/1".into(),
16845            number: 1,
16846            state: "open".into(),
16847            checks: "unknown".into(),
16848            round: 0,
16849            rounds: 1,
16850            red_at_merge: Vec::new(),
16851        });
16852        runner.sync_to_base().await.unwrap();
16853        assert_eq!(runner.state.status, RunStatus::Blocked);
16854        assert_eq!(
16855            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
16856            crate::base_sync_recovery::StopKind::ClosingIntegratedPr
16857        );
16858        assert!(runner.state.base_sync_closure_pending());
16859        assert!(crate::handover::can_resume(&runner.state));
16860        let queue = crate::queue::Queue::open();
16861        assert!(!crate::base_sync_recovery::held_task(
16862            &queue,
16863            &queue.get(&task).unwrap()
16864        ));
16865        assert!(runner.state.authorize_base_sync(&task).is_err());
16866        // Legacy closure stops have no typed stop but must retain the checkpoint.
16867        runner.state.base_sync_recovery.stop = None;
16868        runner.state.status = RunStatus::Voting;
16869        runner.state.gate_ran = true;
16870        runner.state.config.graph.review_rounds = 0;
16871        runner.state.save().unwrap();
16872        assert!(crate::handover::can_resume(&runner.state));
16873        runner.state = RunState::load(&runner.state.id).unwrap();
16874        runner.execute_graph().await.unwrap();
16875        assert_eq!(runner.state.status, RunStatus::AlreadyInBase);
16876        assert_eq!(runner.state.pr.as_ref().unwrap().state, "closed");
16877        assert_eq!(
16878            runner.state.base_sync.as_ref().unwrap().attempts,
16879            BASE_SYNC_ROUNDS
16880        );
16881        assert!(runner.state.base_sync_recovery.authorizations.is_empty());
16882        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), head);
16883        assert!(runner.state.reviews.is_empty());
16884        assert!(runner.state.gate.is_empty());
16885    }
16886
16887    #[cfg(unix)]
16888    #[tokio::test]
16889    async fn base_recovery_authorization_requires_current_owner_free_lease_and_clean_tree() {
16890        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
16891        runner.state.driver_exited = true;
16892        runner.state.save().unwrap();
16893        let home = crate::run::home();
16894        let queue = crate::queue::Queue::open();
16895        let _claim = queue.claim(&task).unwrap();
16896        let lease = crate::driver::maintenance(&home, &runner.state.id).unwrap();
16897        assert!(
16898            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16899                .await
16900                .is_err()
16901        );
16902        drop(lease);
16903        std::fs::write(runner.state.repo.join("dirty"), "retained uncommitted work").unwrap();
16904        assert!(
16905            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16906                .await
16907                .is_err()
16908        );
16909        std::fs::remove_file(runner.state.repo.join("dirty")).unwrap();
16910        Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16911            .await
16912            .unwrap();
16913        let disk = RunState::load(&runner.state.id).unwrap();
16914        assert!(disk.base_recovery_pending());
16915        assert_eq!(disk.base_sync_recovery.authorizations.len(), 1);
16916        assert!(
16917            Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16918                .await
16919                .is_err()
16920        );
16921        assert_eq!(
16922            RunState::load(&runner.state.id)
16923                .unwrap()
16924                .base_sync_recovery
16925                .authorizations
16926                .len(),
16927            1
16928        );
16929        let mut owner = queue.get(&task).unwrap();
16930        owner.runs.push("20260101-000000-ffff".into());
16931        queue.put(&mut owner).unwrap();
16932        let error = Runner::authorize_base_recovery(&home, &runner.state.id, &task)
16933            .await
16934            .unwrap_err();
16935        assert!(error.to_string().contains("run task ownership changed"));
16936    }
16937
16938    #[cfg(unix)]
16939    #[tokio::test]
16940    async fn base_recovery_legacy_voting_and_repeated_resume_stay_blocked() {
16941        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
16942        let mut legacy = serde_json::to_value(&runner.state).unwrap();
16943        legacy.as_object_mut().unwrap().remove("base_sync_recovery");
16944        legacy["status"] = serde_json::json!("voting");
16945        runner.state = serde_json::from_value(legacy).unwrap();
16946        runner.state.save().unwrap();
16947        for _ in 0..3 {
16948            runner.execute_graph().await.unwrap();
16949            assert_eq!(runner.state.status, RunStatus::Blocked);
16950            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
16951            assert!(!crate::handover::can_resume(&runner.state));
16952            assert_eq!(
16953                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16954                head
16955            );
16956            assert!(runner.state.reviews.is_empty());
16957            assert!(!runner.state.gate_ran);
16958        }
16959        let notice = crate::notices::run_ended(&runner.state).unwrap();
16960        let stop = runner.state.base_sync_recovery.stop.clone().unwrap();
16961        runner.state.begin_pass();
16962        runner.state.status = RunStatus::Voting;
16963        runner.deliberate().await.unwrap();
16964        runner.vote().await.unwrap();
16965        runner.finish_after_tally().await.unwrap();
16966        assert_eq!(runner.state.status, RunStatus::Blocked);
16967        assert_eq!(runner.state.base_sync_recovery.stop.as_ref(), Some(&stop));
16968        assert_eq!(
16969            crate::notices::run_ended(&runner.state).unwrap().since,
16970            notice.since
16971        );
16972    }
16973
16974    #[cfg(unix)]
16975    #[tokio::test]
16976    async fn base_recovery_never_authorizes_conflicts_publication_or_unknown_stops() {
16977        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
16978        for detail in [
16979            "fix/retained conflicts with origin/main and did not rebase: conflict",
16980            "rebased fix/retained locally but origin refused the push",
16981            "unknown failure",
16982        ] {
16983            runner.state.base_sync.as_mut().unwrap().conflict = Some(detail.into());
16984            runner.state.record_base_sync_stop(&head);
16985            assert!(runner.state.authorize_base_sync(&task).is_err());
16986            runner.state.status = RunStatus::Voting;
16987            runner.execute_graph().await.unwrap();
16988            assert_eq!(runner.state.status, RunStatus::Blocked);
16989            assert!(!crate::handover::can_resume(&runner.state));
16990            assert_eq!(
16991                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
16992                head
16993            );
16994        }
16995    }
16996
16997    #[cfg(unix)]
16998    #[tokio::test]
16999    async fn base_recovery_consumption_survives_crash_without_another_attempt() {
17000        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17001        runner.state.authorize_base_sync(&task).unwrap();
17002        assert!(runner.state.consume_base_recovery(&head));
17003        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17004        runner.state.reserve_base_operation(&head, &tip);
17005        runner.state.save().unwrap();
17006        runner.state = RunState::load(&runner.state.id).unwrap();
17007        assert!(!runner.state.consume_base_recovery(&head));
17008        runner.sync_to_base().await.unwrap();
17009        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17010        assert_eq!(runner.state.base_episode_used(), 1);
17011        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17012        assert!(
17013            runner
17014                .state
17015                .base_sync_recovery
17016                .operations
17017                .last()
17018                .unwrap()
17019                .finished
17020        );
17021        assert!(!runner.state.base_sync_blocked());
17022        runner.sync_to_base().await.unwrap();
17023        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17024    }
17025
17026    #[cfg(unix)]
17027    #[tokio::test]
17028    async fn base_recovery_legacy_authorization_and_unrecorded_wait_grant_no_interval() {
17029        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17030        // Both historical diagnostic spellings retain an exhaustion identity.
17031        let original = runner
17032            .state
17033            .base_sync
17034            .as_ref()
17035            .unwrap()
17036            .conflict
17037            .clone()
17038            .unwrap();
17039        runner.state.base_sync.as_mut().unwrap().conflict = Some(
17040            original
17041                .replace("synchronization(s)", "rebase(s)")
17042                .replace("synchronizing", "rebasing"),
17043        );
17044        runner.state.base_sync_recovery = Default::default();
17045        runner.state.record_base_sync_stop(&head);
17046        assert_eq!(
17047            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17048            crate::base_sync_recovery::StopKind::Exhausted
17049        );
17050        runner.state.authorize_base_sync(&task).unwrap();
17051        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17052        let auth = legacy["base_sync_recovery"]["authorizations"][0]
17053            .as_object_mut()
17054            .unwrap();
17055        auth.remove("interval");
17056        auth.remove("scope");
17057        runner.state = serde_json::from_value(legacy).unwrap();
17058        assert!(runner.state.consume_base_recovery(&head));
17059        assert_eq!(runner.state.base_episode_used(), 3);
17060        runner.sync_to_base().await.unwrap();
17061        assert_eq!(runner.state.base_episode_used(), 4);
17062        runner.state.resource_block = Some(crate::run::ResourceBlock {
17063            node: "gate".into(),
17064            kind: "cache".into(),
17065            detail: "legacy wait".into(),
17066            at: Timestamp::now(),
17067            cause: crate::run::ResourceBlockCause::Unrecorded,
17068        });
17069        runner.state.save().unwrap();
17070        assert!(
17071            runner
17072                .state
17073                .base_sync_recovery
17074                .episodes
17075                .last()
17076                .unwrap()
17077                .boundary
17078                .is_none()
17079        );
17080        runner.state.resource_block = None;
17081        advance_base_fixture(&runner).await;
17082        runner.sync_to_base().await.unwrap();
17083        assert!(runner.state.base_sync_blocked());
17084        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17085    }
17086
17087    #[cfg(unix)]
17088    async fn advance_base_fixture(runner: &Runner) {
17089        let repo = &runner.state.repo;
17090        for args in [
17091            vec!["checkout", "main"],
17092            vec!["commit", "--allow-empty", "-m", "base advanced"],
17093            vec!["push", "origin", "main"],
17094            vec!["checkout", "fix/retained"],
17095        ] {
17096            git::git(repo, &args).await.unwrap();
17097        }
17098    }
17099
17100    #[cfg(unix)]
17101    fn fresh_base_interval_fixture(runner: &mut Runner) {
17102        runner.state.base_sync.as_mut().unwrap().attempts = 0;
17103        runner.state.base_sync.as_mut().unwrap().conflict = None;
17104        runner.state.base_sync_recovery = Default::default();
17105        runner.state.status = RunStatus::Voting;
17106    }
17107
17108    #[cfg(unix)]
17109    #[tokio::test]
17110    async fn base_recovery_resource_boundary_requires_new_owner_and_is_single_use() {
17111        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17112        fresh_base_interval_fixture(&mut runner);
17113        runner.state.begin_pass();
17114        runner.sync_to_base().await.unwrap();
17115        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17116        runner.state.resource_block = Some(crate::run::ResourceBlock {
17117            node: "gate".into(),
17118            kind: "build cache".into(),
17119            detail: "busy".into(),
17120            at: Timestamp::now(),
17121            cause: crate::run::ResourceBlockCause::Timeout {
17122                waited_secs: 1,
17123                limit_secs: 1,
17124            },
17125        });
17126        runner.state.save().unwrap();
17127        let queue = crate::queue::Queue::open();
17128        let mut owner = queue.get(&task).unwrap();
17129        runner.state.task_instruction = Some(owner.instruction.clone());
17130        owner.release_operator("fixture-owner", None);
17131        queue.put(&mut owner).unwrap();
17132        let approval = owner.resumption.clone().unwrap();
17133        runner.state.resource_block = None;
17134        runner.state.save().unwrap();
17135        runner.state = RunState::load(&runner.state.id).unwrap();
17136        advance_base_fixture(&runner).await;
17137        runner.sync_to_base().await.unwrap();
17138        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17139        assert_eq!(runner.state.base_episode_used(), 1);
17140        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17141        assert!(
17142            runner.state.base_sync_recovery.episodes[0]
17143                .boundary
17144                .as_ref()
17145                .unwrap()
17146                .consumed
17147        );
17148        assert!(
17149            !runner
17150                .state
17151                .renew_base_episode(&task, &approval, &head, "base")
17152        );
17153        for _ in 0..3 {
17154            owner.release_operator("fixture-owner", None);
17155            queue.put(&mut owner).unwrap();
17156            runner.state = RunState::load(&runner.state.id).unwrap();
17157            runner.state.begin_pass();
17158            runner.sync_to_base().await.unwrap();
17159            assert_eq!(runner.state.base_sync_recovery.episodes.len(), 2);
17160            assert_eq!(runner.state.base_episode_used(), 1);
17161        }
17162        runner.state.resource_block = Some(crate::run::ResourceBlock {
17163            node: "gate".into(),
17164            kind: "disk pressure".into(),
17165            detail: "free space required".into(),
17166            at: Timestamp::now(),
17167            cause: crate::run::ResourceBlockCause::Other,
17168        });
17169        runner.state.end_pass_parked(ParkCause::Interrupt {
17170            reason: "disk pressure".into(),
17171        });
17172        runner.state.save().unwrap();
17173        owner.release_operator("fixture-owner-after-disk", None);
17174        queue.put(&mut owner).unwrap();
17175        runner.state.resource_block = None;
17176        advance_base_fixture(&runner).await;
17177        runner.sync_to_base().await.unwrap();
17178        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 3);
17179        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 3);
17180        assert_eq!(runner.state.base_episode_used(), 1);
17181        assert_eq!(runner.state.base_sync_recovery.renewals.len(), 2);
17182    }
17183
17184    #[cfg(unix)]
17185    #[tokio::test]
17186    async fn base_recovery_continuous_advancement_stops_at_four() {
17187        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
17188        fresh_base_interval_fixture(&mut runner);
17189        for spent in 1..=BASE_SYNC_ROUNDS {
17190            runner.sync_to_base().await.unwrap();
17191            assert!(!runner.state.base_sync_blocked());
17192            assert_eq!(runner.state.base_episode_used(), spent);
17193            advance_base_fixture(&runner).await;
17194        }
17195        runner.sync_to_base().await.unwrap();
17196        assert_eq!(
17197            runner.state.base_sync_recovery.stop.as_ref().unwrap().kind,
17198            crate::base_sync_recovery::StopKind::Exhausted
17199        );
17200        let stop = runner.state.base_sync_recovery.stop.as_ref().unwrap();
17201        assert!(stop.unverified_sync);
17202        let action = runner.state.base_sync_next_action();
17203        assert!(action.contains("never reached its gate"));
17204        assert!(action.contains("4/4 in episode 0, 4 lifetime"));
17205        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17206        runner.state = RunState::load(&runner.state.id).unwrap();
17207        runner.sync_to_base().await.unwrap();
17208        assert_eq!(runner.state.base_episode_used(), 4);
17209        assert_eq!(runner.state.base_sync_recovery.episodes.len(), 1);
17210    }
17211
17212    #[cfg(unix)]
17213    #[tokio::test]
17214    async fn base_recovery_stop_diagnostic_requires_synced_head_without_current_gate() {
17215        let (_tmp, mut runner, head, _) = exhausted_base_fixture().await;
17216        runner.state.ensure_base_episode();
17217        // Verification progress alone is not evidence of a successful sync.
17218        runner.state.record_base_progress(&head);
17219        runner.state.base_sync_recovery.stop = None;
17220        runner.state.record_base_sync_stop(&head);
17221        assert!(
17222            !runner
17223                .state
17224                .base_sync_recovery
17225                .stop
17226                .as_ref()
17227                .unwrap()
17228                .unverified_sync
17229        );
17230
17231        runner.state.finish_base_operation(Some(&head));
17232        // A gate for another head cannot verify this synchronization.
17233        runner.state.gate_ran = true;
17234        runner.state.gate_head = Some("older-head".into());
17235        runner.state.base_sync_recovery.stop = None;
17236        runner.state.record_base_sync_stop(&head);
17237        runner.state.save().unwrap();
17238        runner.state = RunState::load(&runner.state.id).unwrap();
17239        assert!(
17240            runner
17241                .state
17242                .base_sync_recovery
17243                .stop
17244                .as_ref()
17245                .unwrap()
17246                .unverified_sync
17247        );
17248
17249        runner.state.gate_head = Some(head.clone());
17250        runner.state.base_sync_recovery.stop = None;
17251        runner.state.record_base_sync_stop(&head);
17252        assert!(
17253            !runner
17254                .state
17255                .base_sync_recovery
17256                .stop
17257                .as_ref()
17258                .unwrap()
17259                .unverified_sync
17260        );
17261        assert!(
17262            !runner
17263                .state
17264                .base_sync_next_action()
17265                .contains("never reached its gate")
17266        );
17267
17268        // Historical records cannot establish the missing synchronization evidence.
17269        let mut legacy = serde_json::to_value(&runner.state).unwrap();
17270        legacy["base_sync_recovery"]["stop"]
17271            .as_object_mut()
17272            .unwrap()
17273            .remove("unverified_sync");
17274        legacy["base_sync_recovery"]["episodes"][0]
17275            .as_object_mut()
17276            .unwrap()
17277            .remove("last_sync_head");
17278        runner.state = serde_json::from_value(legacy).unwrap();
17279        assert!(
17280            !runner
17281                .state
17282                .base_sync_recovery
17283                .stop
17284                .as_ref()
17285                .unwrap()
17286                .unverified_sync
17287        );
17288        assert!(
17289            runner.state.base_sync_recovery.episodes[0]
17290                .last_sync_head
17291                .is_none()
17292        );
17293    }
17294
17295    #[cfg(unix)]
17296    #[tokio::test]
17297    async fn base_recovery_next_action_reports_new_episode_and_pending_approval() {
17298        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17299        runner.state.authorize_base_sync(&task).unwrap();
17300        assert!(
17301            runner
17302                .state
17303                .base_sync_next_action()
17304                .contains("Approval recorded")
17305        );
17306        assert!(
17307            runner
17308                .state
17309                .base_sync_next_action()
17310                .contains("4/4 in episode 0, 4 lifetime")
17311        );
17312        assert!(runner.state.consume_base_recovery(&head));
17313        assert!(
17314            runner
17315                .state
17316                .base_sync_next_action()
17317                .contains("0/4 in episode 1, 4 lifetime")
17318        );
17319    }
17320
17321    #[cfg(unix)]
17322    #[tokio::test]
17323    async fn base_recovery_changed_head_invalidates_without_spending() {
17324        let (_tmp, mut runner, _, task) = exhausted_base_fixture().await;
17325        runner.state.authorize_base_sync(&task).unwrap();
17326        git::git(
17327            &runner.state.repo,
17328            &["commit", "--allow-empty", "-m", "external head change"],
17329        )
17330        .await
17331        .unwrap();
17332        runner.sync_to_base().await.unwrap();
17333        let a = runner
17334            .state
17335            .base_sync_recovery
17336            .authorizations
17337            .last()
17338            .unwrap();
17339        assert!(a.consumed);
17340        assert!(a.invalidated.as_deref().unwrap().contains("HEAD changed"));
17341        runner.state = RunState::load(&runner.state.id).unwrap();
17342        assert!(
17343            runner
17344                .state
17345                .base_sync_next_action()
17346                .contains("Approval invalidated")
17347        );
17348        assert!(!runner.state.base_recovery_pending());
17349        assert!(runner.state.base_sync_blocked());
17350        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17351    }
17352
17353    #[cfg(unix)]
17354    #[tokio::test]
17355    async fn base_recovery_stale_fetched_base_invalidates_approval() {
17356        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17357        runner.state.driver_exited = true;
17358        runner.state.save().unwrap();
17359        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17360            .await
17361            .unwrap();
17362        runner.state = RunState::load(&runner.state.id).unwrap();
17363        advance_base_fixture(&runner).await;
17364        runner.sync_to_base().await.unwrap();
17365        let a = runner
17366            .state
17367            .base_sync_recovery
17368            .authorizations
17369            .last()
17370            .unwrap();
17371        assert!(
17372            a.consumed
17373                && a.invalidated
17374                    .as_deref()
17375                    .unwrap()
17376                    .contains("fetched base changed")
17377        );
17378        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 4);
17379        assert_eq!(
17380            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17381            head
17382        );
17383        Runner::authorize_base_recovery(&crate::run::home(), &runner.state.id, &task)
17384            .await
17385            .unwrap();
17386        runner.state = RunState::load(&runner.state.id).unwrap();
17387        runner.sync_to_base().await.unwrap();
17388        assert!(!runner.state.base_sync_blocked());
17389        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17390    }
17391
17392    #[cfg(unix)]
17393    #[tokio::test]
17394    async fn base_recovery_crash_after_git_finishes_reserved_operation_once() {
17395        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17396        runner.state.authorize_base_sync(&task).unwrap();
17397        assert!(runner.state.consume_base_recovery(&head));
17398        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17399        runner.state.reserve_base_operation(&head, &tip);
17400        runner.state.save().unwrap();
17401        let scratch = runner.state.dir().join("base-sync");
17402        assert_eq!(
17403            crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17404                .await
17405                .unwrap(),
17406            crate::rebase::Rebased::Applied
17407        );
17408        runner.state = RunState::load(&runner.state.id).unwrap();
17409        runner.sync_to_base().await.unwrap();
17410        assert!(
17411            runner
17412                .state
17413                .base_sync_recovery
17414                .operations
17415                .last()
17416                .unwrap()
17417                .finished
17418        );
17419        assert_eq!(runner.state.base_episode_used(), 1);
17420        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17421        assert!(!runner.state.base_sync_blocked());
17422    }
17423
17424    #[cfg(unix)]
17425    #[tokio::test]
17426    async fn base_recovery_crash_preserves_untracked_files() {
17427        for ignored in [false, true] {
17428            let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17429            runner.state.authorize_base_sync(&task).unwrap();
17430            assert!(runner.state.consume_base_recovery(&head));
17431            let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17432            runner.state.reserve_base_operation(&head, &tip);
17433            runner.state.save().unwrap();
17434            let scratch = runner.state.dir().join("base-sync");
17435            assert_eq!(
17436                crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17437                    .await
17438                    .unwrap(),
17439                crate::rebase::Rebased::Applied
17440            );
17441            let repo = &runner.state.repo;
17442            let result = git::rev_parse(repo, "HEAD").await.unwrap();
17443            assert_ne!(result, head);
17444            if ignored {
17445                let exclude = git::git(repo, &["rev-parse", "--git-path", "info/exclude"])
17446                    .await
17447                    .unwrap();
17448                std::fs::write(repo.join(exclude), "local-notes\n").unwrap();
17449            }
17450            std::fs::write(repo.join("local-notes"), "retained work\n").unwrap();
17451            assert!(
17452                git::git_raw(repo, &["diff", "--quiet", &head])
17453                    .await
17454                    .unwrap()
17455                    .ok()
17456            );
17457            // Resume after the branch moved, before its checkout was reset.
17458            runner.state = RunState::load(&runner.state.id).unwrap();
17459            runner.sync_to_base().await.unwrap();
17460            assert!(runner.state.base_sync_blocked());
17461            assert!(!runner.state.base_sync_recovery.operations[0].finished);
17462            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17463            assert_eq!(
17464                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17465                result
17466            );
17467            assert_eq!(
17468                std::fs::read_to_string(runner.state.repo.join("local-notes")).unwrap(),
17469                "retained work\n"
17470            );
17471            assert!(
17472                git::git_raw(&runner.state.repo, &["diff", "--quiet", &head])
17473                    .await
17474                    .unwrap()
17475                    .ok()
17476            );
17477        }
17478    }
17479
17480    #[cfg(unix)]
17481    #[tokio::test]
17482    async fn base_recovery_crash_does_not_adopt_foreign_commits() {
17483        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
17484        runner.state.authorize_base_sync(&task).unwrap();
17485        assert!(runner.state.consume_base_recovery(&head));
17486        let tip = runner.state.base_sync.as_ref().unwrap().tip.clone();
17487        runner.state.reserve_base_operation(&head, &tip);
17488        runner.state.save().unwrap();
17489        let scratch = runner.state.dir().join("base-sync");
17490        crate::rebase::rebase_with_fixer(&mut runner.state, &scratch, "fix/retained", &tip)
17491            .await
17492            .unwrap();
17493        git::sync_to_head(&runner.state.repo).await.unwrap();
17494        std::fs::write(runner.state.repo.join("foreign"), "foreign work").unwrap();
17495        git::git(&runner.state.repo, &["add", "foreign"])
17496            .await
17497            .unwrap();
17498        git::git(&runner.state.repo, &["commit", "-m", "foreign work"])
17499            .await
17500            .unwrap();
17501        let foreign_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17502        runner.state = RunState::load(&runner.state.id).unwrap();
17503        runner.sync_to_base().await.unwrap();
17504        assert!(runner.state.base_sync_blocked());
17505        assert!(
17506            !runner
17507                .state
17508                .base_sync_recovery
17509                .operations
17510                .last()
17511                .unwrap()
17512                .finished
17513        );
17514        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
17515        assert_eq!(
17516            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17517            foreign_head
17518        );
17519    }
17520
17521    #[cfg(unix)]
17522    async fn paused_base_conflict_fixture() -> (tempfile::TempDir, Runner, String, String, PathBuf)
17523    {
17524        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17525        fresh_base_interval_fixture(&mut runner);
17526        let repo = &runner.state.repo;
17527        git::git(repo, &["checkout", "main"]).await.unwrap();
17528        std::fs::write(repo.join("retained"), "upstream change\n").unwrap();
17529        git::git(repo, &["add", "retained"]).await.unwrap();
17530        git::git(repo, &["commit", "-m", "upstream retained change"])
17531            .await
17532            .unwrap();
17533        git::git(repo, &["push", "origin", "main"]).await.unwrap();
17534        let tip = git::rev_parse(repo, "HEAD").await.unwrap();
17535        git::git(repo, &["checkout", "fix/retained"]).await.unwrap();
17536        let scratch = runner.state.dir().join("base-sync");
17537        assert!(matches!(
17538            git::rebase_start(repo, &scratch, "fix/retained", &tip)
17539                .await
17540                .unwrap(),
17541            git::RebaseStart::Conflicted(_)
17542        ));
17543        runner.state.rebase_fixes.push(crate::run::RebaseFixRecord {
17544            agent: "fixture".into(),
17545            paths: vec!["retained".into()],
17546            from: Some(head.clone()),
17547            stop: Some(git::rev_parse(&scratch, "REBASE_HEAD").await.unwrap()),
17548            index_outside: None,
17549            finished: false,
17550            error: None,
17551            launcher: None,
17552            infra: None,
17553            outage: None,
17554        });
17555        (tmp, runner, head, tip, scratch)
17556    }
17557
17558    #[cfg(unix)]
17559    #[tokio::test]
17560    async fn base_recovery_imports_legacy_rebase_before_following_new_main() {
17561        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17562        // A resource stop from before reservations existed, with resolved
17563        // contents left for the launcher's ordinary staging continuation.
17564        std::fs::write(
17565            scratch.join("retained"),
17566            "upstream and retained resolution\n",
17567        )
17568        .unwrap();
17569        runner.state.resource_block = Some(crate::run::ResourceBlock {
17570            node: "rebase".into(),
17571            kind: "repository coordination".into(),
17572            detail: "busy".into(),
17573            at: Timestamp::now(),
17574            cause: crate::run::ResourceBlockCause::Other,
17575        });
17576        runner.state.save().unwrap();
17577        // Advance the remote without moving either checkout participating
17578        // in the standing rebase.
17579        let newer = git::git(
17580            &runner.state.repo,
17581            &[
17582                "commit-tree",
17583                "main^{tree}",
17584                "-p",
17585                "main",
17586                "-m",
17587                "base advanced during wait",
17588            ],
17589        )
17590        .await
17591        .unwrap();
17592        git::git(
17593            &runner.state.repo,
17594            &[
17595                "push",
17596                "origin",
17597                &format!("{}:refs/heads/main", newer.trim()),
17598            ],
17599        )
17600        .await
17601        .unwrap();
17602        runner.state = RunState::load(&runner.state.id).unwrap();
17603        runner.state.resource_block = None;
17604        runner.sync_to_base().await.unwrap();
17605        assert!(!runner.state.base_sync_blocked());
17606        let op = &runner.state.base_sync_recovery.operations[0];
17607        assert_eq!(op.head, head);
17608        assert_eq!(op.tip, tip);
17609        assert!(op.finished);
17610        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17611        assert_eq!(runner.state.rebase_fixes.len(), 1);
17612        let latest = git::rev_parse(&runner.state.repo, "origin/main")
17613            .await
17614            .unwrap();
17615        assert_ne!(latest, tip);
17616        runner.state = RunState::load(&runner.state.id).unwrap();
17617        runner.sync_to_base().await.unwrap();
17618        assert!(!runner.state.base_sync_blocked());
17619        assert_eq!(runner.state.base_sync_recovery.operations.len(), 2);
17620        assert_eq!(runner.state.base_sync_recovery.operations[1].tip, latest);
17621        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 2);
17622    }
17623
17624    #[cfg(unix)]
17625    #[tokio::test]
17626    async fn base_recovery_resumes_exact_validated_conflict_result_after_crash() {
17627        let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17628        runner.state.reserve_base_operation(&head, &tip);
17629        runner.state.save().unwrap();
17630        std::fs::write(
17631            scratch.join("retained"),
17632            "upstream and retained resolution\n",
17633        )
17634        .unwrap();
17635        git::git(&scratch, &["add", "retained"]).await.unwrap();
17636        git::git(
17637            &scratch,
17638            &["-c", "core.editor=true", "rebase", "--continue"],
17639        )
17640        .await
17641        .unwrap();
17642        let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17643        assert!(
17644            !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17645                .await
17646                .unwrap()
17647                .is_empty()
17648        );
17649        assert_eq!(
17650            crate::rebase::finish(
17651                &mut runner.state,
17652                &scratch,
17653                "fix/retained",
17654                &head,
17655                &tip,
17656                &["retained".into()],
17657                ""
17658            )
17659            .await
17660            .unwrap(),
17661            crate::rebase::Rebased::Applied
17662        );
17663        assert!(!scratch.exists());
17664        // Simulate losing the caller before finish_base_operation/save.
17665        runner.state = RunState::load(&runner.state.id).unwrap();
17666        let op = &runner.state.base_sync_recovery.operations[0];
17667        assert!(!op.finished);
17668        assert_eq!(op.validated_head.as_deref(), Some(result.as_str()));
17669        runner.sync_to_base().await.unwrap();
17670        assert!(!runner.state.base_sync_blocked());
17671        assert!(runner.state.base_sync_recovery.operations[0].finished);
17672        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17673        assert_eq!(runner.state.rebase_fixes.len(), 1);
17674        let winner = runner.state.winner().unwrap().clone();
17675        let wait = runner.repository_wait();
17676        assert!(
17677            verify_current_head(&mut runner.state, &winner, &wait)
17678                .await
17679                .unwrap()
17680        );
17681        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17682    }
17683
17684    #[cfg(unix)]
17685    #[tokio::test]
17686    async fn base_recovery_revalidates_completed_scratch_before_validation_save() {
17687        for damage in [None, Some("record"), Some("foreign"), Some("dirty")] {
17688            let (_tmp, mut runner, head, tip, scratch) = paused_base_conflict_fixture().await;
17689            runner.state.reserve_base_operation(&head, &tip);
17690            runner.state.save().unwrap();
17691            std::fs::write(
17692                scratch.join("retained"),
17693                "upstream and retained resolution\n",
17694            )
17695            .unwrap();
17696            git::git(&scratch, &["add", "retained"]).await.unwrap();
17697            git::git(
17698                &scratch,
17699                &["-c", "core.editor=true", "rebase", "--continue"],
17700            )
17701            .await
17702            .unwrap();
17703            let result = git::rev_parse(&scratch, "HEAD").await.unwrap();
17704            assert!(!git::rebase_in_progress(&scratch).await);
17705            assert!(
17706                !crate::reconcile::origin_missing(&runner.state.repo, &result, &head)
17707                    .await
17708                    .unwrap()
17709                    .is_empty()
17710            );
17711            match damage {
17712                Some("record") => {
17713                    let path =
17714                        git::git(&scratch, &["rev-parse", "--git-path", "magi-rebase-heads"])
17715                            .await
17716                            .unwrap();
17717                    std::fs::write(scratch.join(path), "").unwrap();
17718                }
17719                Some("foreign") => {
17720                    std::fs::write(scratch.join("foreign"), "foreign work").unwrap();
17721                    git::git(&scratch, &["add", "foreign"]).await.unwrap();
17722                    git::git(&scratch, &["commit", "-m", "foreign work"])
17723                        .await
17724                        .unwrap();
17725                }
17726                Some("dirty") => {
17727                    std::fs::write(scratch.join("retained"), "uncommitted work").unwrap();
17728                }
17729                _ => {}
17730            }
17731            let retained_head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17732            // Crash immediately after Git continuation, before finish runs.
17733            runner.state = RunState::load(&runner.state.id).unwrap();
17734            assert!(
17735                runner.state.base_sync_recovery.operations[0]
17736                    .validated_head
17737                    .is_none()
17738            );
17739            runner.sync_to_base().await.unwrap();
17740            assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17741            assert_eq!(runner.state.base_episode_used(), 1);
17742            assert_eq!(runner.state.rebase_fixes.len(), 1);
17743            assert_eq!(
17744                git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
17745                retained_head
17746            );
17747            if damage.is_some() {
17748                assert!(runner.state.base_sync_blocked());
17749                assert!(scratch.exists());
17750                assert!(!runner.state.base_sync_recovery.operations[0].finished);
17751                assert!(
17752                    runner.state.base_sync_recovery.operations[0]
17753                        .validated_head
17754                        .is_none()
17755                );
17756            } else {
17757                assert!(!runner.state.base_sync_blocked());
17758                assert!(!scratch.exists());
17759                assert!(runner.state.base_sync_recovery.operations[0].finished);
17760                assert_eq!(
17761                    runner.state.base_sync_recovery.operations[0]
17762                        .validated_head
17763                        .as_deref(),
17764                    Some(result.as_str())
17765                );
17766                let winner = runner.state.winner().unwrap().clone();
17767                let wait = runner.repository_wait();
17768                assert!(
17769                    verify_current_head(&mut runner.state, &winner, &wait)
17770                        .await
17771                        .unwrap()
17772                );
17773                assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, result);
17774                runner.state = RunState::load(&runner.state.id).unwrap();
17775                runner.sync_to_base().await.unwrap();
17776                assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 1);
17777            }
17778        }
17779    }
17780
17781    #[cfg(unix)]
17782    async fn resumed_gate_fixture() -> (tempfile::TempDir, Runner, String) {
17783        let (tmp, mut runner, head, _) = exhausted_base_fixture().await;
17784        // This is an ordinary gate stop, not an authorized base-budget recovery.
17785        runner.state.prep_complete = Some(true);
17786        runner.state.base_sync = None;
17787        runner.state.base_sync_recovery = Default::default();
17788        runner.state.config.graph.gate_fix_rounds = 1;
17789        let mut review = review_round(true, 0, 1, 1, false, true);
17790        review.head = head.clone();
17791        review.verified_head = Some(head.clone());
17792        runner.state.reviews = vec![review];
17793        runner.state.gate = vec![CommandOutcome {
17794            command: "old gate".into(),
17795            code: Some(1),
17796            output_tail: "old failure".into(),
17797            duration_ms: 0,
17798            resource_blocked: false,
17799        }];
17800        runner.state.gate_head = Some(head.clone());
17801        runner.state.gate_ran = true;
17802        runner.state.gate_fixes = vec![crate::run::GateFixRecord {
17803            agent: "fixture".into(),
17804            failed: runner.state.gate.clone(),
17805            notes: "spent".into(),
17806            committed: false,
17807            error: None,
17808        }];
17809        runner.state.land_approval = Some(crate::run::LandApproval {
17810            question: "old approval".into(),
17811            head: head.clone(),
17812        });
17813        runner.state.status = RunStatus::Blocked;
17814        (tmp, runner, head)
17815    }
17816
17817    #[cfg(unix)]
17818    #[tokio::test]
17819    async fn resumed_gate_rebase_verifies_new_head_without_refilling_repairs() {
17820        for (e2e_ok, gate_ok) in [(true, true), (true, false), (false, true)] {
17821            let (tmp, mut runner, old) = resumed_gate_fixture().await;
17822            let phases = tmp.path().join("phases");
17823            runner.state.config.verify.e2e = vec![format!(
17824                "echo e2e >> '{}'; {}",
17825                phases.display(),
17826                if e2e_ok {
17827                    "true"
17828                } else {
17829                    "echo new-e2e-failure; false"
17830                }
17831            )];
17832            runner.state.config.verify.gate = vec![format!(
17833                "echo gate >> '{}'; {}",
17834                phases.display(),
17835                if gate_ok {
17836                    "true"
17837                } else {
17838                    "echo new-gate-failure; false"
17839                }
17840            )];
17841            runner.state.save().unwrap();
17842            runner.state = RunState::load(&runner.state.id).unwrap();
17843            runner.sync_to_base().await.unwrap();
17844            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17845            assert_ne!(head, old);
17846            // Reentry after Git updated but before verification was invalidated.
17847            runner.state = RunState::load(&runner.state.id).unwrap();
17848            runner.execute_graph().await.unwrap();
17849            assert_eq!(runner.state.gate_fixes.len(), 1);
17850            assert_eq!(runner.state.reviews.len(), 1);
17851            assert_eq!(runner.state.reviews[0].head, old);
17852            assert_eq!(runner.state.land_approval.as_ref().unwrap().head, old);
17853            assert_eq!(runner.state.verification_history.len(), 1);
17854            let history = &runner.state.verification_history[0];
17855            assert_eq!(history.gate_head.as_deref(), Some(old.as_str()));
17856            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17857            assert!(history.archived_at.is_some());
17858            assert_eq!(history.fixes_spent, Some(1));
17859            assert_eq!(history.gate[0].output_tail, "old failure");
17860            let archive_events: Vec<_> = runner
17861                .state
17862                .events
17863                .iter()
17864                .filter(|e| e.node == "gate" && e.message.starts_with("archived gate on "))
17865                .collect();
17866            assert_eq!(archive_events.len(), 1);
17867            assert!(archive_events[0].message.contains(&short(&old)));
17868            assert!(archive_events[0].message.contains(&short(&head)));
17869            assert!(
17870                archive_events[0]
17871                    .message
17872                    .contains("1 gate-fix round(s) already spent stay spent")
17873            );
17874            assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, head);
17875            assert_eq!(runner.state.head_e2e.as_ref().unwrap().passed, e2e_ok);
17876            if e2e_ok {
17877                assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17878                assert_eq!(runner.state.gate_allows_progress(), gate_ok);
17879            } else {
17880                assert!(!runner.state.gate_ran);
17881            }
17882            assert_eq!(
17883                runner.state.status,
17884                if e2e_ok && gate_ok {
17885                    RunStatus::Ready
17886                } else {
17887                    RunStatus::Blocked
17888                }
17889            );
17890            if !e2e_ok || !gate_ok {
17891                let cause = runner.state.verification_blocking_cause().unwrap();
17892                assert!(cause.contains(&head));
17893                assert!(cause.contains(if e2e_ok { "gate failed" } else { "e2e failed" }));
17894                // A later successful progress event must not become the cause.
17895                runner.state.event("land", "rebased successfully");
17896                let notice = crate::notices::run_ended(&runner.state).unwrap();
17897                assert!(!format!("{notice:?}").contains("rebased successfully"));
17898                assert!(runner.state.merge.is_none());
17899            }
17900            let before = std::fs::read_to_string(&phases).unwrap();
17901            runner.state.save().unwrap();
17902            runner.state = RunState::load(&runner.state.id).unwrap();
17903            runner.review_loop().await.unwrap();
17904            runner.gate().await.unwrap();
17905            assert_eq!(std::fs::read_to_string(&phases).unwrap(), before);
17906            assert_eq!(runner.state.verification_history.len(), 1);
17907            let history = &runner.state.verification_history[0];
17908            assert_eq!(history.superseded_by.as_deref(), Some(head.as_str()));
17909            assert!(history.archived_at.is_some());
17910            assert_eq!(history.fixes_spent, Some(1));
17911        }
17912    }
17913
17914    #[cfg(unix)]
17915    #[tokio::test]
17916    async fn resumed_gate_unchanged_head_keeps_failure_and_unknown_head_rechecks() {
17917        let (tmp, mut runner, head) = resumed_gate_fixture().await;
17918        let marker = tmp.path().join("gate-ran");
17919        runner.state.config.verify.gate = vec![format!("touch '{}'", marker.display())];
17920        runner.review_loop().await.unwrap();
17921        runner.gate().await.unwrap();
17922        assert_eq!(runner.state.status, RunStatus::Blocked);
17923        assert!(!marker.exists());
17924        assert!(runner.state.verification_history.is_empty());
17925        runner.state.gate_head = None;
17926        runner.gate().await.unwrap();
17927        assert!(marker.exists());
17928        assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
17929        assert_eq!(runner.state.verification_history.len(), 1);
17930        assert_eq!(runner.state.gate_fixes.len(), 1);
17931    }
17932
17933    #[cfg(unix)]
17934    #[tokio::test]
17935    async fn resumed_gate_pending_e2e_retries_without_spending_or_landing() {
17936        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
17937        runner.sync_to_base().await.unwrap();
17938        let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17939        runner.state.reconcile_verification_head(&head);
17940        // Both crash-before-command and resource wait are unresolved evidence.
17941        for outcomes in [
17942            Vec::new(),
17943            vec![CommandOutcome {
17944                command: "true".into(),
17945                code: None,
17946                output_tail: "cache unavailable".into(),
17947                duration_ms: 0,
17948                resource_blocked: true,
17949            }],
17950        ] {
17951            runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17952                head: head.clone(),
17953                passed: false,
17954                outcomes,
17955            });
17956            runner.state.gate_ran = false;
17957            runner.state.gate.clear();
17958            runner.state.save().unwrap();
17959            runner.state = RunState::load(&runner.state.id).unwrap();
17960            runner.merge().await.unwrap();
17961            assert!(runner.state.merge.is_none());
17962            runner.review_loop().await.unwrap();
17963            runner.gate().await.unwrap();
17964            assert!(runner.state.head_e2e.as_ref().unwrap().passed);
17965            assert!(runner.state.gate_allows_progress());
17966            assert_eq!(runner.state.gate_fixes.len(), 1);
17967            assert!(runner.state.merge.is_none());
17968        }
17969        // Moving the winner after a successful gate cannot publish an unchecked SHA.
17970        git::git(
17971            &runner.state.repo,
17972            &["commit", "--allow-empty", "-m", "external update"],
17973        )
17974        .await
17975        .unwrap();
17976        runner.merge().await.unwrap();
17977        assert!(runner.state.merge.is_none());
17978        assert_eq!(runner.state.status, RunStatus::Blocked);
17979    }
17980
17981    #[cfg(unix)]
17982    #[tokio::test]
17983    async fn resumed_gate_resource_waits_preserve_budget_and_retry_on_reentry() {
17984        for waiting_on_gate in [false, true] {
17985            let (tmp, mut runner, _old) = resumed_gate_fixture().await;
17986            runner.sync_to_base().await.unwrap();
17987            let head = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
17988            runner.state.reconcile_verification_head(&head);
17989            if waiting_on_gate {
17990                runner.state.head_e2e = Some(crate::run::GateRepairE2e {
17991                    head: head.clone(),
17992                    passed: true,
17993                    outcomes: Vec::new(),
17994                });
17995            }
17996            let cache = tmp.path().join("cache");
17997            runner.state.config.verify.e2e =
17998                vec![format!("CARGO_TARGET_DIR='{}' true", cache.display())];
17999            runner.state.config.verify.gate = vec!["true".into()];
18000            runner.state.config.graph.timeout_resource_wait = 1;
18001            let owner =
18002                crate::cache::Owner::here("other", "e2e", "e2e", &runner.state.repo, "other-head");
18003            let guard =
18004                match crate::cache::try_acquire(&crate::run::home(), &cache, &owner).unwrap() {
18005                    crate::cache::AcquireOutcome::Acquired(g) => g,
18006                    crate::cache::AcquireOutcome::Busy(b) => panic!("unexpected busy cache: {b:?}"),
18007                };
18008            runner.review_loop().await.unwrap();
18009            runner.gate().await.unwrap();
18010            assert!(!runner.state.gate_ran);
18011            assert_eq!(runner.state.gate_fixes.len(), 1);
18012            assert!(runner.state.resource_block.is_some());
18013            runner.merge().await.unwrap();
18014            assert!(runner.state.merge.is_none());
18015            drop(guard);
18016            runner.state = RunState::load(&runner.state.id).unwrap();
18017            // The driver clears the previous pass's resource stop on resume.
18018            runner.state.resource_block = None;
18019            runner.review_loop().await.unwrap();
18020            runner.gate().await.unwrap();
18021            assert!(runner.state.gate_allows_progress());
18022            assert_eq!(runner.state.gate_head.as_deref(), Some(head.as_str()));
18023            assert_eq!(runner.state.gate_fixes.len(), 1);
18024        }
18025    }
18026
18027    #[cfg(unix)]
18028    #[tokio::test]
18029    async fn resumed_gate_pre_gate_head_change_and_review_holds() {
18030        let (_tmp, mut runner, _old) = resumed_gate_fixture().await;
18031        runner.sync_to_base().await.unwrap();
18032        let before = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18033        runner.state.config.verify.pre_gate = vec!["echo mechanical > mechanical".into()];
18034        // Verification passes before pre_gate and fails on the mechanically changed head.
18035        runner.state.config.verify.e2e = vec!["test ! -f mechanical".into()];
18036        runner.review_loop().await.unwrap();
18037        runner.gate().await.unwrap();
18038        let after = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18039        assert_ne!(before, after);
18040        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, after);
18041        assert!(!runner.state.head_e2e.as_ref().unwrap().passed);
18042        assert!(!runner.state.gate_allows_progress());
18043        runner.merge().await.unwrap();
18044        assert!(runner.state.merge.is_none());
18045        assert!(
18046            runner
18047                .state
18048                .verification_history
18049                .iter()
18050                .any(|h| h.e2e.as_ref().is_some_and(|e| e.head == before && e.passed))
18051        );
18052
18053        // An incomplete review is still an independent hold on a changed head.
18054        runner.state.reviews[0].clean = false;
18055        runner.state.reviews[0].answered = 0;
18056        runner.state.config.verify.e2e = vec!["true".into()];
18057        runner.review_loop().await.unwrap();
18058        runner.gate().await.unwrap();
18059        runner.merge().await.unwrap();
18060        assert_eq!(runner.state.status, RunStatus::Blocked);
18061        assert!(runner.state.merge.is_none());
18062        assert!(
18063            runner
18064                .state
18065                .verification_blocking_cause()
18066                .unwrap()
18067                .contains("review blocked")
18068        );
18069    }
18070
18071    #[cfg(unix)]
18072    #[tokio::test]
18073    async fn resumed_gate_cannot_certify_a_head_changed_by_its_command() {
18074        let (_tmp, mut runner, old) = resumed_gate_fixture().await;
18075        runner.state.gate_head = None;
18076        runner.state.config.verify.gate =
18077            vec!["git -c commit.gpgsign=false commit --allow-empty -m 'gate changed head'".into()];
18078        runner.review_loop().await.unwrap();
18079        runner.gate().await.unwrap();
18080        assert_ne!(
18081            git::rev_parse(&runner.state.repo, "HEAD").await.unwrap(),
18082            old
18083        );
18084        assert_eq!(runner.state.gate_head.as_deref(), Some(old.as_str()));
18085        assert!(!runner.state.gate_allows_progress());
18086        assert_eq!(runner.state.status, RunStatus::Blocked);
18087        runner.merge().await.unwrap();
18088        assert!(runner.state.merge.is_none());
18089    }
18090
18091    #[cfg(unix)]
18092    #[tokio::test]
18093    async fn base_recovery_success_requires_new_review_e2e_and_gate() {
18094        let (_tmp, mut runner, head, task) = exhausted_base_fixture().await;
18095        let mut old_review = review_round(true, 0, 1, 1, false, true);
18096        old_review.head = head.clone();
18097        old_review.verified_head = Some(head.clone());
18098        runner.state.reviews = vec![old_review];
18099        runner.state.gate = vec![CommandOutcome {
18100            command: "old gate".into(),
18101            code: None,
18102            output_tail: "timed out after 2400s".into(),
18103            duration_ms: 2_400_000,
18104            resource_blocked: false,
18105        }];
18106        runner.state.gate_ran = true;
18107        runner.state.gate_head = Some(head.clone());
18108        runner.state.land_approval = Some(crate::run::LandApproval {
18109            question: "old-head-approval".into(),
18110            head: head.clone(),
18111        });
18112        runner.state.authorize_base_sync(&task).unwrap();
18113        runner.sync_to_base().await.unwrap();
18114        assert!(!runner.state.base_sync_blocked());
18115        let fresh = git::rev_parse(&runner.state.repo, "HEAD").await.unwrap();
18116        assert_ne!(head, fresh);
18117        assert_eq!(runner.state.reviews.len(), 1);
18118        assert!(!runner.state.gate_ran);
18119        let saved = &runner.state.base_sync_recovery.authorizations[0];
18120        assert_eq!(saved.reviews[0].head, head);
18121        assert_eq!(saved.gate_head.as_deref(), Some(head.as_str()));
18122        assert!(saved.gate_ran);
18123        assert!(!saved.gate[0].ok());
18124        runner.review_loop().await.unwrap();
18125        runner.gate().await.unwrap();
18126        assert_eq!(runner.state.reviews[0].head, head);
18127        assert_eq!(
18128            runner.state.reviews[0].verified_head.as_deref(),
18129            Some(head.as_str())
18130        );
18131        assert_eq!(runner.state.head_e2e.as_ref().unwrap().head, fresh);
18132        assert!(runner.state.head_e2e.as_ref().unwrap().passed);
18133        assert_eq!(runner.state.gate_head.as_deref(), Some(fresh.as_str()));
18134        assert!(runner.state.gate_ran);
18135        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
18136        assert_eq!(runner.state.land_approval.as_ref().unwrap().head, head);
18137        assert_ne!(runner.state.land_approval.as_ref().unwrap().head, fresh);
18138        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 5);
18139        assert_eq!(
18140            std::fs::read_to_string(runner.state.repo.join("retained")).unwrap(),
18141            "retained change"
18142        );
18143    }
18144
18145    #[cfg(unix)]
18146    #[tokio::test]
18147    async fn base_recovery_moving_base_uses_bounded_authorized_interval() {
18148        let (_tmp, mut runner, _head, task) = exhausted_base_fixture().await;
18149        let queue = crate::queue::Queue::open();
18150        let mut owner = queue.get(&task).unwrap();
18151        owner.release_operator("test-resume", None);
18152        queue.put(&mut owner).unwrap();
18153        let generation = owner.resumption.as_ref().unwrap().generation;
18154        runner.state.task_instruction = Some(owner.instruction.clone());
18155        runner.state.driver_exited = true;
18156        runner.state.save().unwrap();
18157        let home = crate::run::home();
18158        let _claim = queue.claim(&task).unwrap();
18159        owner.resumption.as_mut().unwrap().at =
18160            runner.state.base_sync_recovery.stop.as_ref().unwrap().at;
18161        queue.put(&mut owner).unwrap();
18162        assert!(
18163            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18164                .await
18165                .is_err(),
18166            "an approval from before the stop grants nothing"
18167        );
18168        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18169        queue.put(&mut owner).unwrap();
18170        Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18171            .await
18172            .unwrap();
18173        assert!(
18174            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18175                .await
18176                .is_err()
18177        );
18178        runner.state = RunState::load(&runner.state.id).unwrap();
18179        assert_eq!(
18180            runner.state.base_sync_recovery.authorizations[0].generation,
18181            Some(generation)
18182        );
18183        runner.sync_to_base().await.unwrap();
18184        let repo = runner.state.repo.clone();
18185        let head = git::rev_parse(&repo, "HEAD").await.unwrap();
18186        git::git(&repo, &["checkout", "main"]).await.unwrap();
18187        std::fs::write(repo.join("base-change"), "base moved again").unwrap();
18188        git::git(&repo, &["add", "."]).await.unwrap();
18189        git::git(&repo, &["commit", "-m", "move base again"])
18190            .await
18191            .unwrap();
18192        git::git(&repo, &["push", "origin", "main"]).await.unwrap();
18193        git::git(&repo, &["checkout", "fix/retained"])
18194            .await
18195            .unwrap();
18196        runner.sync_to_base().await.unwrap();
18197        assert!(!runner.state.base_sync_blocked());
18198        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18199        assert_eq!(runner.state.base_episode_used(), 2);
18200        let fresh = git::rev_parse(&repo, "HEAD").await.unwrap();
18201        assert_ne!(fresh, head);
18202        assert!(!runner.state.base_recovery_pending());
18203        // Even replaying the generation with a later timestamp grants nothing.
18204        owner.resumption.as_mut().unwrap().at = Timestamp::now();
18205        queue.put(&mut owner).unwrap();
18206        assert!(
18207            Runner::authorize_base_recovery_from_resumption(&home, &runner.state.id, &task)
18208                .await
18209                .is_err()
18210        );
18211        runner.state = RunState::load(&runner.state.id).unwrap();
18212        runner.sync_to_base().await.unwrap();
18213        assert_eq!(runner.state.base_sync.as_ref().unwrap().attempts, 6);
18214        assert_eq!(runner.state.base_sync_recovery.authorizations.len(), 1);
18215        assert_eq!(git::rev_parse(&repo, "HEAD").await.unwrap(), fresh);
18216    }
18217
18218    #[cfg(unix)]
18219    #[tokio::test]
18220    async fn base_recovery_tally_skips_deliberation_and_voting_without_a_sync_stop() {
18221        let (_tmp, mut runner, _, _) = exhausted_base_fixture().await;
18222        runner.state.base_sync = None;
18223        runner.state.status = RunStatus::Blocked;
18224        let mut second = runner.state.candidates[0].clone();
18225        second.label = 'B';
18226        runner.state.candidates.push(second);
18227        assert!(runner.state.votes.is_empty());
18228        assert!(runner.state.deliberation.is_empty());
18229        runner.deliberate().await.unwrap();
18230        runner.vote().await.unwrap();
18231        assert_eq!(runner.state.status, RunStatus::Blocked);
18232        assert!(runner.state.votes.is_empty());
18233        assert!(runner.state.deliberation.is_empty());
18234    }
18235
18236    /// A minimal, git-free `Runner` at a given status — `settle_questions`
18237    /// reads nothing else off it.
18238    fn runner_at(status: RunStatus) -> Runner {
18239        let mut state = RunState::new(
18240            PathBuf::from("/nonexistent/repo"),
18241            "main".to_owned(),
18242            "deadbeef".to_owned(),
18243            "task".to_owned(),
18244            Config::default(),
18245        );
18246        // These fixtures test execution flow, independent of host disk capacity.
18247        state.config.disk.min_free_bytes = 0;
18248        // These fixtures share one home and save concurrently. Short clock
18249        // IDs can collide, making an ownership test read another test's run.
18250        static NEXT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
18251        state.id = format!(
18252            "20000101-000000-{:04x}",
18253            NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
18254        );
18255        state.status = status;
18256        Runner {
18257            state,
18258            roles: ResolvedRoles {
18259                implementers: Vec::new(),
18260                judges: Vec::new(),
18261                reviewers: Vec::new(),
18262                fixer: None,
18263                conductor: conductor(),
18264                implementer_roster: Vec::new(),
18265                judge_roster: Vec::new(),
18266                reviewer_roster: Vec::new(),
18267            },
18268            sem: Arc::new(Semaphore::new(1)),
18269            pause: Pause::new(),
18270            interrupt: Pause::new(),
18271        }
18272    }
18273
18274    #[tokio::test]
18275    async fn source_eviction_resume_restores_before_graph_shortcuts_and_setup() {
18276        crate::run::pin_test_home();
18277        let temp = tempfile::tempdir().unwrap();
18278        let root = temp.path().canonicalize().unwrap();
18279        let repo = root.join("repo");
18280        std::fs::create_dir(&repo).unwrap();
18281        init_repo(&repo);
18282        git::git(&repo, &["config", "core.autocrlf", "false"])
18283            .await
18284            .unwrap();
18285        let mut runner = runner_at(RunStatus::Blocked);
18286        runner.state.repo = repo.clone();
18287        runner.state.base_commit = git::rev_parse(&repo, "HEAD").await.unwrap();
18288        runner.state.config.agents = vec![conductor()];
18289        runner.state.config.graph.selected_implementer = Some("conductor".into());
18290        runner.state.config.graph.worktree_root = Some(root.join("worktrees"));
18291        let (shell, command, setup_output) = if cfg!(windows) {
18292            (
18293                vec!["cmd".into(), "/C".into()],
18294                "echo ready>seat-ready",
18295                "ready\r\n",
18296            )
18297        } else {
18298            (
18299                vec!["sh".into(), "-c".into()],
18300                "printf ready > seat-ready",
18301                "ready",
18302            )
18303        };
18304        runner.state.config.verify.shell = Some(shell);
18305        runner.state.config.worktree.setup = vec![crate::config::SetupStep {
18306            run: Some(command.into()),
18307            ..Default::default()
18308        }];
18309        runner.state.save().unwrap();
18310        let path = runner.state.worktree_root().join("judge-1");
18311        git::worktree_add_detached(&repo, &path, &runner.state.base_commit)
18312            .await
18313            .unwrap();
18314        std::fs::write(path.join("checkpoint"), "unrelated seat").unwrap();
18315        git::git(&path, &["add", "."]).await.unwrap();
18316        let tree = git::git(&path, &["write-tree"]).await.unwrap();
18317        let head = git::git(&path, &["commit-tree", &tree, "-m", "seat checkpoint"])
18318            .await
18319            .unwrap();
18320        git::git(&path, &["checkout", "--detach", &head])
18321            .await
18322            .unwrap();
18323        let evidence = crate::fold_evidence::snapshot(
18324            &runner.state,
18325            &crate::run::home().canonicalize().unwrap(),
18326            &head,
18327            64 * 1024 * 1024,
18328        )
18329        .await
18330        .unwrap();
18331        let mut contents = BTreeMap::new();
18332        contents.insert(
18333            repo.join("logs/HEAD"),
18334            format!("{head} {head} fixture\n").into_bytes(),
18335        );
18336        let record = serde_json::json!({
18337            "version":1, "run":runner.state.id, "source":path, "head":head,
18338            "evidence":evidence, "admin":{}, "admin_contents":contents,
18339            "history":{head.clone():evidence}, "phase":"evicted"
18340        });
18341        std::fs::write(
18342            runner.state.dir().join("artifacts/eviction-judge-1.json"),
18343            serde_json::to_vec(&record).unwrap(),
18344        )
18345        .unwrap();
18346        assert!(git::worktree_remove_clean(&repo, &path).await.unwrap());
18347        git::git(&repo, &["reflog", "expire", "--expire=now", "--all"])
18348            .await
18349            .unwrap();
18350        git::git(&repo, &["gc", "--prune=now"]).await.unwrap();
18351        assert!(
18352            !git::git_raw(&repo, &["cat-file", "-e", &head])
18353                .await
18354                .unwrap()
18355                .ok()
18356        );
18357        let mut resumed = Runner::resume(&runner.state.id).unwrap();
18358        let error = resumed.execute().await.unwrap_err();
18359        assert!(
18360            error
18361                .to_string()
18362                .contains("selected implementation is held"),
18363            "{error:#}"
18364        );
18365        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), head);
18366        assert_eq!(
18367            std::fs::read_to_string(path.join("seat-ready")).unwrap(),
18368            setup_output
18369        );
18370        assert_eq!(
18371            std::fs::read_to_string(path.join("checkpoint")).unwrap(),
18372            "unrelated seat"
18373        );
18374        let error = resumed.execute().await.unwrap_err();
18375        assert!(
18376            error
18377                .to_string()
18378                .contains("selected implementation is held"),
18379            "{error:#}"
18380        );
18381        std::fs::write(path.join("checkpoint"), "next recorded review").unwrap();
18382        git::git(&path, &["add", "."]).await.unwrap();
18383        let next_tree = git::git(&path, &["write-tree"]).await.unwrap();
18384        let next = git::git(
18385            &path,
18386            &[
18387                "commit-tree",
18388                &next_tree,
18389                "-p",
18390                &head,
18391                "-m",
18392                "next review checkpoint",
18393            ],
18394        )
18395        .await
18396        .unwrap();
18397        git::git(&path, &["checkout", "--detach", &next])
18398            .await
18399            .unwrap();
18400        resumed.state.reviews.push(
18401            serde_json::from_value(serde_json::json!({
18402                "round":1, "head":next, "reviews":[]
18403            }))
18404            .unwrap(),
18405        );
18406        resumed.state.save().unwrap();
18407        let error = resumed.execute().await.unwrap_err();
18408        assert!(
18409            error
18410                .to_string()
18411                .contains("selected implementation is held"),
18412            "{error:#}"
18413        );
18414        assert_eq!(git::rev_parse(&path, "HEAD").await.unwrap(), next);
18415    }
18416
18417    #[tokio::test]
18418    async fn driver_claim_waits_for_maintenance_before_starting_a_pass() {
18419        crate::run::pin_test_home();
18420        let mut runner = runner_at(RunStatus::Gating);
18421        runner.state.save().unwrap();
18422        let id = runner.state.id.clone();
18423        let passes = runner.state.passes.len();
18424        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18425        let mut claiming = Box::pin(runner.claim_driver());
18426        std::future::poll_fn(|cx| {
18427            assert!(std::future::Future::poll(claiming.as_mut(), cx).is_pending());
18428            std::task::Poll::Ready(())
18429        })
18430        .await;
18431        assert_eq!(RunState::load(&id).unwrap().passes.len(), passes);
18432        drop(owner);
18433        let execution = tokio::time::timeout(Duration::from_secs(2), claiming)
18434            .await
18435            .unwrap()
18436            .unwrap();
18437        assert_eq!(execution.runner.state.passes.len(), passes + 1);
18438        assert!(execution.runner.state.resource_block.is_none());
18439        assert_eq!(
18440            execution.runner.state.liveness(false),
18441            crate::run::Liveness::Live
18442        );
18443        drop(execution);
18444        assert!(RunState::load(&id).unwrap().driver_exited);
18445    }
18446
18447    #[tokio::test]
18448    async fn a_failed_driver_lease_record_releases_the_claim_and_retained_state() {
18449        crate::run::pin_test_home();
18450        let mut runner = runner_at(RunStatus::Gating);
18451        runner.state.save().unwrap();
18452        // run.json can commit, but the subsequent lease metadata write fails.
18453        // Cleanup must run after releasing the short ownership lock.
18454        std::fs::create_dir(runner.state.dir().join("driver.json.tmp")).unwrap();
18455        assert!(runner.claim_driver().await.is_err());
18456        assert!(runner.state.driver_exited);
18457        assert_eq!(runner.state.driver_pid, None);
18458        let disk = RunState::load(&runner.state.id).unwrap();
18459        assert!(disk.driver_exited);
18460        assert_eq!(disk.driver_pid, None);
18461        assert_eq!(disk.status, RunStatus::Gating);
18462    }
18463
18464    #[tokio::test]
18465    async fn an_early_graph_error_releases_its_driver() {
18466        crate::run::pin_test_home();
18467        let mut runner = runner_at(RunStatus::Blocked);
18468        runner.state.config.graph.selected_implementer = Some("selected".into());
18469        runner.state.save().unwrap();
18470        assert!(runner.execute().await.is_err());
18471        let disk = RunState::load(&runner.state.id).unwrap();
18472        assert!(disk.driver_exited);
18473        assert_eq!(disk.driver_pid, None);
18474        assert_eq!(disk.status, RunStatus::Blocked);
18475    }
18476
18477    fn merge_repair_request(phase: crate::run::MergeRepairPhase) -> crate::run::MergeRepair {
18478        crate::run::MergeRepair {
18479            question: "repair-question".into(),
18480            head: "old-head".into(),
18481            pr: "https://github.com/o/r/pull/42".into(),
18482            reviews: vec![review_round(false, 1, 1, 1, false, true)],
18483            gate: vec![outcome(Some(0), false)],
18484            gate_head: Some("old-head".into()),
18485            gate_fixes: Vec::new(),
18486            pre_gate: Vec::new(),
18487            phase,
18488            failure: None,
18489        }
18490    }
18491
18492    #[tokio::test]
18493    async fn merge_repair_claim_refuses_concurrent_and_obsolete_drivers() {
18494        crate::run::pin_test_home();
18495        let mut runner = runner_at(RunStatus::Landing);
18496        runner.state.merge_repairs.push(merge_repair_request(
18497            crate::run::MergeRepairPhase::Requested,
18498        ));
18499        runner.state.save().unwrap();
18500        let claim = FixClaim::acquire(&runner.state.dir()).unwrap();
18501        assert!(
18502            runner
18503                .execute()
18504                .await
18505                .unwrap_err()
18506                .to_string()
18507                .contains("already running")
18508        );
18509        assert_eq!(
18510            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18511            crate::run::MergeRepairPhase::Requested
18512        );
18513        drop(claim);
18514        let mut disk = runner.state.clone();
18515        disk.merge_repairs[0].phase = crate::run::MergeRepairPhase::Complete;
18516        disk.save().unwrap();
18517        assert!(
18518            runner
18519                .execute()
18520                .await
18521                .unwrap_err()
18522                .to_string()
18523                .contains("decision changed")
18524        );
18525        assert_eq!(
18526            RunState::load(&runner.state.id).unwrap().merge_repairs[0].phase,
18527            crate::run::MergeRepairPhase::Complete
18528        );
18529    }
18530
18531    #[test]
18532    fn merge_repair_starts_a_real_new_budget_and_preserves_old_evidence() {
18533        crate::run::pin_test_home();
18534        let mut runner = runner_at(RunStatus::Landing);
18535        let request = merge_repair_request(crate::run::MergeRepairPhase::Fixing);
18536        runner.state.reviews = request.reviews.clone();
18537        runner.state.gate = request.gate.clone();
18538        runner.state.gate_ran = true;
18539        runner.state.land_approval = Some(crate::run::LandApproval {
18540            question: "old".into(),
18541            head: "old-head".into(),
18542        });
18543        runner.state.merge_repairs.push(request);
18544        runner.begin_merge_repair_review(0).unwrap();
18545        assert_eq!(review_conclusion(&runner.state.reviews, 2), None);
18546        assert!(!runner.state.gate_status().ok());
18547        assert!(runner.state.land_approval.is_none());
18548        let disk = RunState::load(&runner.state.id).unwrap();
18549        assert_eq!(disk.merge_repairs[0].reviews[0].blocking, 1);
18550        assert!(disk.merge_repairs[0].gate[0].ok());
18551        assert_eq!(
18552            disk.merge_repairs[0].phase,
18553            crate::run::MergeRepairPhase::Reviewing
18554        );
18555        assert_eq!(disk.merge_repairs[0].pr, "https://github.com/o/r/pull/42");
18556        assert!(disk.reviews.is_empty());
18557    }
18558
18559    #[tokio::test]
18560    async fn merge_repair_interrupted_fixer_and_exhausted_generation_hold_without_repeating_work() {
18561        crate::run::pin_test_home();
18562        for phase in [
18563            crate::run::MergeRepairPhase::Fixing,
18564            crate::run::MergeRepairPhase::Requested,
18565        ] {
18566            let mut runner = runner_at(RunStatus::Landing);
18567            runner.state.config.graph.land_rounds = 0;
18568            runner.state.merge_repairs.push(merge_repair_request(phase));
18569            runner.state.save().unwrap();
18570            assert!(!runner.repair_merge_review().await.unwrap());
18571            assert_eq!(runner.state.status, RunStatus::Blocked);
18572            assert_eq!(
18573                runner.state.merge_repairs[0].phase,
18574                crate::run::MergeRepairPhase::Failed
18575            );
18576            let reason = runner.state.merge_repairs[0].failure.as_ref().unwrap();
18577            assert!(
18578                reason.contains(if phase == crate::run::MergeRepairPhase::Fixing {
18579                    "interrupted"
18580                } else {
18581                    "budget exhausted"
18582                })
18583            );
18584            assert!(runner.state.merge.is_none());
18585        }
18586    }
18587
18588    /// `park_here` folding in the reason `Pause::park_because` recorded -
18589    /// this is what lets an operator reading a run's events tell an
18590    /// interrupt-driven park from an ordinary shutdown park.
18591    #[test]
18592    fn park_here_folds_the_interrupt_reason_into_the_park_event() {
18593        crate::run::pin_test_home();
18594        let mut runner = runner_at(RunStatus::Implementing);
18595        let interrupt = Pause::new();
18596        runner.watch_interrupt(interrupt.clone());
18597
18598        interrupt.park_because("task a1b2 asked to run first");
18599
18600        assert!(runner.park_here().expect("park_here"));
18601        assert!(runner.state.parked);
18602        let last = runner.state.events.last().expect("a park event");
18603        assert_eq!(last.node, "park");
18604        assert!(
18605            last.message.contains("task a1b2 asked to run first"),
18606            "expected the interrupt reason in {:?}",
18607            last.message
18608        );
18609    }
18610
18611    /// The cause is stored on the pass when it parks, and the first cause a
18612    /// shared pause was given is the one that stays.
18613    #[test]
18614    fn park_here_records_the_structured_cause_and_the_first_one_wins() {
18615        crate::run::pin_test_home();
18616        let mut runner = runner_at(RunStatus::Implementing);
18617        runner.state.begin_pass();
18618        let shutdown = Pause::new();
18619        runner.on_pause(shutdown.clone());
18620        let upgrade = ParkCause::Upgrade {
18621            from: "0.121.0".to_owned(),
18622            to: Some("0.121.1".to_owned()),
18623        };
18624        shutdown.park_for(upgrade.clone());
18625        shutdown.park_for(ParkCause::Operator);
18626
18627        assert!(runner.park_here().expect("park_here"));
18628        assert_eq!(
18629            runner.state.passes[0].end,
18630            Some(crate::run::PassEnd::Parked { cause: upgrade })
18631        );
18632        let last = runner.state.events.last().expect("a park event");
18633        assert!(last.message.contains("0.121.0"), "{}", last.message);
18634    }
18635
18636    #[tokio::test]
18637    async fn driver_admission_accepts_a_freshly_minted_record_under_its_lease() {
18638        crate::run::pin_test_home();
18639        let mut runner = runner_at(RunStatus::Reviewing);
18640        runner.state.save().unwrap();
18641        let mut execution = runner.claim_driver().await.expect("first execution");
18642        assert_eq!(execution.runner.state.passes.len(), 1);
18643        execution
18644            .driver
18645            .finish(&mut execution.runner.state)
18646            .await
18647            .unwrap();
18648    }
18649
18650    #[tokio::test]
18651    async fn driver_admission_resumes_a_record_saved_before_its_first_claim() {
18652        crate::run::pin_test_home();
18653        let mut runner = runner_at(RunStatus::Reviewing);
18654        runner.state.config.agents = vec![conductor()];
18655        runner.state.config.graph.timeout_resource_wait = 0;
18656        runner.state.save().unwrap();
18657        let id = runner.state.id.clone();
18658        let owner = crate::pr_head::owner_lock(&crate::run::home(), &id).unwrap();
18659        assert!(runner.claim_driver().await.is_err());
18660        drop(owner);
18661        drop(runner);
18662        let mut resumed = Runner::resume(&id).unwrap();
18663        assert_eq!(resumed.state.driver_claimed, Some(false));
18664        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Dead);
18665        assert_eq!(resumed.state.liveness(true), crate::run::Liveness::Live);
18666        let mut execution = resumed
18667            .claim_driver()
18668            .await
18669            .expect("retry initial claim after reload");
18670        assert_eq!(execution.runner.state.driver_claimed, Some(true));
18671        assert_eq!(execution.runner.state.passes.len(), 1);
18672        execution
18673            .driver
18674            .finish(&mut execution.runner.state)
18675            .await
18676            .unwrap();
18677    }
18678
18679    #[tokio::test]
18680    async fn driver_admission_preclaim_marker_does_not_bypass_an_execution_lease() {
18681        crate::run::pin_test_home();
18682        let mut runner = runner_at(RunStatus::Reviewing);
18683        runner.state.config.agents = vec![conductor()];
18684        runner.state.save().unwrap();
18685        let id = runner.state.id.clone();
18686        let lease = crate::driver::maintenance(&crate::run::home(), &id).unwrap();
18687        let mut resumed = Runner::resume(&id).unwrap();
18688        assert_eq!(resumed.state.liveness(false), crate::run::Liveness::Live);
18689        assert!(resumed.claim_driver().await.is_err());
18690        assert!(RunState::load(&id).unwrap().passes.is_empty());
18691        drop(lease);
18692        let mut execution = resumed.claim_driver().await.unwrap();
18693        execution
18694            .driver
18695            .finish(&mut execution.runner.state)
18696            .await
18697            .unwrap();
18698    }
18699
18700    #[tokio::test]
18701    async fn driver_admission_accepts_legacy_unknown_process_with_a_proven_free_pass_lease() {
18702        crate::run::pin_test_home();
18703        let mut runner = runner_at(RunStatus::Reviewing);
18704        let state = &mut runner.state;
18705        state.config.agents = vec![conductor()];
18706        state.driver_claimed = None;
18707        state.driver_pid = Some(std::process::id());
18708        state.driver_started_at = None;
18709        state.begin_pass();
18710        state.save().unwrap();
18711        let lease = crate::driver::maintenance(&crate::run::home(), &state.id).unwrap();
18712        std::fs::write(
18713            state.dir().join("driver.json"),
18714            serde_json::to_vec(&serde_json::json!({
18715                "pid": state.driver_pid, "started_at": null,
18716                "pass": state.passes[0].started_at, "pass_index": state.passes[0].index,
18717            }))
18718            .unwrap(),
18719        )
18720        .unwrap();
18721        assert_eq!(state.liveness(false), crate::run::Liveness::Live);
18722        drop(lease);
18723        assert_eq!(state.liveness(false), crate::run::Liveness::Dead);
18724        let mut resumed = Runner::resume(&state.id).unwrap();
18725        assert!(!resumed.state.driver_exited);
18726        let mut execution = resumed
18727            .claim_driver()
18728            .await
18729            .expect("lease proves previous execution ended");
18730        assert_eq!(execution.runner.state.passes.len(), 2);
18731        execution
18732            .driver
18733            .finish(&mut execution.runner.state)
18734            .await
18735            .unwrap();
18736    }
18737
18738    #[tokio::test]
18739    async fn driver_admission_admits_a_legacy_record_without_a_claim_marker() {
18740        crate::run::pin_test_home();
18741        let mut runner = runner_at(RunStatus::Reviewing);
18742        let mut json = serde_json::to_value(&runner.state).unwrap();
18743        json.as_object_mut().unwrap().remove("driver_claimed");
18744        runner.state = serde_json::from_value(json).unwrap();
18745        assert_eq!(runner.state.driver_claimed, None);
18746        runner.state.save().unwrap();
18747        let mut execution = runner
18748            .claim_driver()
18749            .await
18750            .expect("the execution lease, not a missing marker, excludes a second driver");
18751        assert_eq!(execution.runner.state.passes.len(), 1);
18752        execution
18753            .driver
18754            .finish(&mut execution.runner.state)
18755            .await
18756            .unwrap();
18757    }
18758
18759    #[tokio::test]
18760    async fn driver_admission_refuses_a_proven_live_driver() {
18761        crate::run::pin_test_home();
18762        let mut runner = runner_at(RunStatus::Reviewing);
18763        let pid = std::process::id();
18764        runner.state.driver_pid = Some(pid);
18765        runner.state.driver_started_at = crate::proc::process_started_at(pid);
18766        assert!(runner.state.driver_started_at.is_some());
18767        runner.state.save().unwrap();
18768        let error = runner
18769            .claim_driver()
18770            .await
18771            .err()
18772            .expect("a live driver must refuse");
18773        assert!(error.to_string().contains("live driver"), "{error:#}");
18774        let saved = RunState::load(&runner.state.id).unwrap();
18775        assert_eq!(saved.driver_pid, Some(pid));
18776        assert_eq!(saved.status, RunStatus::Reviewing);
18777        assert!(saved.passes.is_empty());
18778    }
18779
18780    #[test]
18781    fn parking_resource_halt_persists_the_pass_once() {
18782        crate::run::pin_test_home();
18783        let mut runner = runner_at(RunStatus::Reviewing);
18784        runner.state.begin_pass();
18785        runner.state.resource_block = Some(crate::run::ResourceBlock {
18786            node: "fix".into(),
18787            kind: "shared build cache".into(),
18788            detail: "cancelled".into(),
18789            at: Timestamp::now(),
18790            cause: crate::run::ResourceBlockCause::Cancelled { park: None },
18791        });
18792        let cause = ParkCause::Upgrade {
18793            from: "old".into(),
18794            to: Some("new".into()),
18795        };
18796        runner.pause.park_for(cause.clone());
18797        // Both direct fixer exits and graph resource boundaries must complete
18798        // parking, rather than returning a stale Reviewing record.
18799        runner.stop_for_resource().unwrap();
18800        assert!(runner.resource_halt().unwrap());
18801        assert!(runner.park_here().unwrap());
18802        let saved = RunState::load(&runner.state.id).unwrap();
18803        assert!(saved.parked);
18804        assert_eq!(saved.status, RunStatus::Reviewing);
18805        assert!(saved.passes[0].ended_at.is_some());
18806        assert_eq!(
18807            saved.passes[0].end,
18808            Some(crate::run::PassEnd::Parked {
18809                cause: cause.clone()
18810            })
18811        );
18812        assert_eq!(
18813            saved.resource_block.unwrap().cause,
18814            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
18815        );
18816        assert_eq!(
18817            saved
18818                .events
18819                .iter()
18820                .filter(|event| event.node == "park")
18821                .count(),
18822            1
18823        );
18824    }
18825
18826    /// `watch_interrupt` and `on_pause` are genuinely independent: an ordinary
18827    /// shutdown `Pause` (what `Stop::park` hands every run, shared and never
18828    /// cleared) must not make a *different* run - one only watching its own,
18829    /// unshared interrupt `Pause` - see itself as parked. If a future change
18830    /// ever collapsed these back into one handle, the interrupt scheduler
18831    /// would park every run for the rest of the daemon's life, not just the
18832    /// one it meant to interrupt.
18833    #[test]
18834    fn the_stop_level_pause_and_a_runs_interrupt_pause_do_not_leak_into_each_other() {
18835        crate::run::pin_test_home();
18836        let mut runner = runner_at(RunStatus::Implementing);
18837        let shutdown = Pause::new();
18838        runner.on_pause(shutdown.clone());
18839        let interrupt = Pause::new();
18840        runner.watch_interrupt(interrupt.clone());
18841
18842        // Nobody has asked for anything yet.
18843        assert!(!runner.park_here().expect("park_here"));
18844        assert!(!runner.state.parked);
18845
18846        // Only the interrupt handle fires; the shutdown handle stays clear.
18847        interrupt.park_because("test");
18848        assert!(!shutdown.parked());
18849        assert!(runner.park_here().expect("park_here"));
18850    }
18851
18852    /// The property every prior attempt at this feature failed to pin down:
18853    /// asking a run to park while one of its nodes has a real, in-flight
18854    /// async operation running (an agent call, in production) must not cut
18855    /// that operation short. `park_here` is only ever consulted *between*
18856    /// `execute`'s node calls - see its own doc - so nothing inside a node
18857    /// can observe a park request until the node itself returns. This proves
18858    /// that structurally, with real `tokio` concurrency and a channel
18859    /// handshake (never a sleep, which would only prove "usually", not
18860    /// "cannot"): the "node" below reports that it has genuinely started,
18861    /// and only then is the park requested; the node still has to be told to
18862    /// finish before `park_here` is ever called, exactly mirroring every
18863    /// `self.some_node().await; if self.park_here()? { return Ok(()); }` pair
18864    /// in `execute`.
18865    #[tokio::test]
18866    async fn a_park_request_made_mid_node_only_takes_effect_at_the_next_boundary() {
18867        crate::run::pin_test_home();
18868        let mut runner = runner_at(RunStatus::Implementing);
18869        let interrupt = Pause::new();
18870        runner.watch_interrupt(interrupt.clone());
18871
18872        let (started_tx, started_rx) = tokio::sync::oneshot::channel::<()>();
18873        let (finish_tx, finish_rx) = tokio::sync::oneshot::channel::<()>();
18874
18875        // Stands in for one node's in-flight agent call: it proves it has
18876        // genuinely started, then blocks - exactly as a spawned CLI process
18877        // does - until told to finish.
18878        let node = async move {
18879            started_tx.send(()).expect("send started");
18880            finish_rx.await.expect("recv finish");
18881            "node finished"
18882        };
18883
18884        let interrupter = async move {
18885            started_rx.await.expect("recv started");
18886            // The call is now genuinely in flight. Ask it to park.
18887            interrupt.park_because("higher-priority task waiting");
18888            // Nothing the node does can observe this yet - there is no
18889            // check inside it, by construction - so let the executor run
18890            // anything pending and then let the node finish on its own.
18891            tokio::task::yield_now().await;
18892            finish_tx.send(()).expect("send finish");
18893        };
18894
18895        let (node_result, ()) = tokio::join!(node, interrupter);
18896        assert_eq!(
18897            node_result, "node finished",
18898            "the in-flight call ran to completion"
18899        );
18900
18901        // Only now, at the boundary the real `execute` would check right
18902        // after this node, does the park take effect.
18903        assert!(runner.park_here().expect("park_here"));
18904        assert!(runner.state.parked);
18905    }
18906
18907    /// A run parked mid-competition carries every field it had accumulated
18908    /// through the exact same disk round-trip an ordinary resume uses -
18909    /// `RunState::save`/`RunState::load`, which is all `Runner::resume` is.
18910    /// Nothing about parking for an interrupt is a special case of that path;
18911    /// this is what proves it rather than assuming it.
18912    #[test]
18913    fn a_run_parked_for_an_interrupt_resumes_with_nothing_lost() {
18914        crate::run::pin_test_home();
18915        let mut runner = runner_at(RunStatus::Judging);
18916        // `Runner::resume` re-resolves roles from the saved config, which
18917        // refuses an empty roster - give it the same minimal one `conductor`
18918        // itself uses.
18919        runner.state.config.agents = vec![conductor()];
18920        runner.state.candidates = vec![Candidate {
18921            index: 0,
18922            label: 'A',
18923            agent: "alpha".to_owned(),
18924            branch: "magi/x/A".to_owned(),
18925            worktree: PathBuf::from("/nonexistent/worktree"),
18926            summary: "did the thing".to_owned(),
18927            stat: "1 file changed".to_owned(),
18928            files: 1,
18929            commits: 1,
18930            empty: false,
18931            failed: None,
18932            verified_noop: None,
18933            duration_ms: 1234,
18934            folded: false,
18935        }];
18936        let run_id = runner.state.id.clone();
18937
18938        let interrupt = Pause::new();
18939        runner.watch_interrupt(interrupt.clone());
18940        interrupt.park_because("task c3d4 asked to run first");
18941        assert!(runner.park_here().expect("park_here"));
18942
18943        let resumed = Runner::resume(&run_id).expect("resume");
18944        assert_eq!(resumed.state.candidates.len(), 1);
18945        assert_eq!(resumed.state.candidates[0].summary, "did the thing");
18946        assert_eq!(resumed.state.candidates[0].branch, "magi/x/A");
18947        assert_eq!(resumed.state.status, runner.state.status);
18948        assert!(
18949            resumed.state.parked,
18950            "still parked until `execute` actually walks the graph again"
18951        );
18952        assert!(resumed.state.events.iter().any(|e| e.node == "park"));
18953    }
18954
18955    /// A fresh open question on `run`, stored and handed back for assertions.
18956    fn ask_open_question(store: &ask::Questions, run: &str) -> ask::Question {
18957        let mut q = ask::Question::new(
18958            run.to_owned(),
18959            "implement".to_owned(),
18960            "impl-A".to_owned(),
18961            "Which storage backend should the cache use?".to_owned(),
18962            String::new(),
18963            vec!["SQLite".to_owned(), "Redis".to_owned()],
18964        );
18965        store.put(&mut q).unwrap();
18966        q
18967    }
18968
18969    #[test]
18970    fn a_failed_runs_open_question_is_abandoned() {
18971        ask_test_home();
18972        let store = ask::Questions::open();
18973        let mut runner = runner_at(RunStatus::Failed);
18974        let run = runner.state.id.clone();
18975        let q = ask_open_question(&store, &run);
18976
18977        runner.settle_questions();
18978
18979        let back = store.get(&q.id).unwrap();
18980        assert!(
18981            !back.status.open(),
18982            "the seat that asked died with the run; nobody is left to read an answer"
18983        );
18984        assert!(
18985            back.detail.contains(&run) && back.detail.contains("failed"),
18986            "the reason names what the run became, not just that it is gone: {}",
18987            back.detail
18988        );
18989    }
18990
18991    #[test]
18992    fn a_merged_runs_open_question_is_abandoned_too() {
18993        ask_test_home();
18994        let store = ask::Questions::open();
18995        // A run that finishes cleanly still leaves nobody to read an answer -
18996        // this is not only a failure-path cleanup.
18997        for status in [RunStatus::Merged, RunStatus::Ready] {
18998            let mut runner = runner_at(status);
18999            let run = runner.state.id.clone();
19000            let q = ask_open_question(&store, &run);
19001
19002            runner.settle_questions();
19003
19004            let back = store.get(&q.id).unwrap();
19005            assert!(
19006                !back.status.open(),
19007                "{status:?} run's question must not outlive the run"
19008            );
19009        }
19010    }
19011
19012    #[test]
19013    fn a_still_resumable_runs_open_question_is_left_alone() {
19014        ask_test_home();
19015        let store = ask::Questions::open();
19016        // `Blocked` and `Stalled` can still be resumed — the candidates, the
19017        // review round and the seat sessions are all still on disk — so a
19018        // question asked mid-round may yet get a real answer from a real
19019        // resume. Sweeping it here would be exactly the failure mode this
19020        // whole feature exists to avoid on the other side.
19021        for status in [RunStatus::Blocked, RunStatus::Stalled] {
19022            let mut runner = runner_at(status);
19023            let run = runner.state.id.clone();
19024            let q = ask_open_question(&store, &run);
19025
19026            runner.settle_questions();
19027
19028            let back = store.get(&q.id).unwrap();
19029            assert!(
19030                back.status.open(),
19031                "{status:?} is still alive; the question must still be waiting"
19032            );
19033        }
19034    }
19035
19036    #[test]
19037    fn settle_questions_never_touches_an_already_answered_question() {
19038        ask_test_home();
19039        let store = ask::Questions::open();
19040        let mut runner = runner_at(RunStatus::Failed);
19041        let run = runner.state.id.clone();
19042        let mut q = ask_open_question(&store, &run);
19043        q.answer(crate::ask::Answer::Choice("SQLite".to_owned()))
19044            .unwrap();
19045        store.put(&mut q).unwrap();
19046
19047        // Called twice, the way a crash-recovered daemon reclaim and the
19048        // graph's own cleanup both can for the same run — `abandon_for_run`
19049        // only ever touches what is still open, so this must be inert both
19050        // times, not merely the second.
19051        runner.settle_questions();
19052        runner.settle_questions();
19053
19054        let back = store.get(&q.id).unwrap();
19055        assert_eq!(
19056            back.status,
19057            ask::QuestionStatus::Answered,
19058            "a real answer is a decision on record, never overwritten by a sweep"
19059        );
19060    }
19061
19062    /// `fold_run(&mut state, drop_winner = false)` is exactly the call
19063    /// `clean::fold_due` makes for a `Ready`/`Failed` run - one that finished
19064    /// without merging, whose winner is still the operator's answer to read.
19065    /// Nothing previously called `fold_run` itself with a real `tally`, so
19066    /// this is the first test to pin down the one distinction the whole
19067    /// automatic-fold feature depends on: the winner's worktree and branch
19068    /// must survive, everything else sharing the run's worktree bay - a
19069    /// loser, standing in for a judge/review worktree too, since `fold_run`'s
19070    /// second sweep treats every non-winner directory under the bay alike -
19071    /// must not.
19072    #[tokio::test]
19073    async fn fold_run_keeps_only_the_winner_when_the_winner_is_not_dropped() {
19074        crate::run::pin_test_home();
19075        let tmp = tempfile::tempdir().expect("tempdir");
19076        let repo = tmp.path().join("repo");
19077        std::fs::create_dir_all(&repo).unwrap();
19078        init_repo(&repo);
19079
19080        let mut config = Config::default();
19081        config.graph.worktree_root = Some(tmp.path().join("wt"));
19082
19083        let mut state = RunState::new(
19084            repo.clone(),
19085            "main".to_owned(),
19086            "deadbeef".to_owned(),
19087            "task".to_owned(),
19088            config,
19089        );
19090        let root = state.worktree_root();
19091        let wt_a = root.join("cand-A");
19092        let wt_b = root.join("cand-B");
19093        git::worktree_add_branch(&repo, &wt_a, "magi/x/A", "main")
19094            .await
19095            .expect("worktree A");
19096        git::worktree_add_branch(&repo, &wt_b, "magi/x/B", "main")
19097            .await
19098            .expect("worktree B");
19099
19100        state.candidates = vec![
19101            Candidate {
19102                index: 0,
19103                label: 'A',
19104                agent: "alpha".to_owned(),
19105                branch: "magi/x/A".to_owned(),
19106                worktree: wt_a.clone(),
19107                summary: String::new(),
19108                stat: String::new(),
19109                files: 0,
19110                commits: 0,
19111                empty: false,
19112                failed: None,
19113                verified_noop: None,
19114                duration_ms: 0,
19115                folded: false,
19116            },
19117            Candidate {
19118                index: 1,
19119                label: 'B',
19120                agent: "beta".to_owned(),
19121                branch: "magi/x/B".to_owned(),
19122                worktree: wt_b.clone(),
19123                summary: String::new(),
19124                stat: String::new(),
19125                files: 0,
19126                commits: 0,
19127                empty: false,
19128                failed: None,
19129                verified_noop: None,
19130                duration_ms: 0,
19131                folded: false,
19132            },
19133        ];
19134        state.tally = Some(Tally {
19135            first_choice: BTreeMap::from([('A', 1)]),
19136            borda: BTreeMap::new(),
19137            winner: 'A',
19138            rankings: 1,
19139            unanimous_initial: true,
19140            deliberated: false,
19141            changed_votes: 0,
19142            unanimous_final: true,
19143            tie_break: None,
19144            judges: 1,
19145            present: 1,
19146            quorum: 1,
19147            met_quorum: true,
19148            uncontested: None,
19149        });
19150        state.status = RunStatus::Ready;
19151
19152        fold_run(&mut state, false, &crate::run::home())
19153            .await
19154            .expect("fold_run");
19155
19156        assert!(wt_a.exists(), "the unmerged winner's worktree survives");
19157        assert!(
19158            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19159            "the unmerged winner's branch survives"
19160        );
19161        assert!(
19162            !state.candidates[0].folded,
19163            "the winner is not marked folded"
19164        );
19165
19166        assert!(!wt_b.exists(), "the loser's worktree is removed");
19167        assert!(
19168            !git::branch_exists(&repo, "magi/x/B").await.unwrap(),
19169            "the loser's branch is removed"
19170        );
19171        assert!(state.candidates[1].folded, "the loser is marked folded");
19172    }
19173
19174    #[tokio::test]
19175    #[cfg(unix)]
19176    async fn fold_run_waits_for_shared_lock_before_removing_worktrees_branches_or_config() {
19177        use std::os::fd::AsRawFd as _;
19178        use std::time::Duration;
19179
19180        let tmp = tempfile::tempdir().unwrap();
19181        let repo = tmp.path().join("repo");
19182        std::fs::create_dir_all(&repo).unwrap();
19183        init_repo(&repo);
19184        let common = repo.join(".git");
19185        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
19186        let home = tmp.path().join("home");
19187        let mut config = Config::default();
19188        config.coordination = crate::config::test_coordination();
19189        config.graph.worktree_root = Some(tmp.path().join("wt"));
19190        config.graph.timeout_resource_wait = 1;
19191        let mut state = RunState::new(
19192            repo.clone(),
19193            "main".into(),
19194            "deadbeef".into(),
19195            "task".into(),
19196            config,
19197        );
19198        let root = state.worktree_root();
19199        let candidate = root.join("cand-A");
19200        let reviewer = root.join("review-1");
19201        let branch = state.branch_for('A');
19202        git::worktree_add_branch(&repo, &candidate, &branch, "main")
19203            .await
19204            .unwrap();
19205        git::worktree_add_detached(&repo, &reviewer, "main")
19206            .await
19207            .unwrap();
19208        git::acquire_worktree_config(&repo).await.unwrap();
19209        state.enabled_worktree_config = true;
19210        state.status = RunStatus::Ready;
19211        state.candidates.push(Candidate {
19212            index: 0,
19213            label: 'A',
19214            agent: "implement".into(),
19215            branch: branch.clone(),
19216            worktree: candidate.clone(),
19217            summary: String::new(),
19218            stat: String::new(),
19219            files: 0,
19220            commits: 0,
19221            empty: false,
19222            failed: None,
19223            verified_noop: None,
19224            duration_ms: 0,
19225            folded: false,
19226        });
19227        state.save_under(&home).unwrap();
19228        let config_before = std::fs::read(common.join("config")).unwrap();
19229        let worktrees_before = git::git(&repo, &["worktree", "list", "--porcelain"])
19230            .await
19231            .unwrap();
19232        // A separate OS handle bypasses the launcher's local gate, exactly as
19233        // an external updater does, and must still prevent all folding work.
19234        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
19235        assert_eq!(
19236            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
19237            0
19238        );
19239        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19240        assert!(
19241            tokio::time::timeout(Duration::from_millis(200), &mut fold)
19242                .await
19243                .is_err()
19244        );
19245        assert!(candidate.exists());
19246        assert!(reviewer.exists());
19247        assert!(git::branch_exists(&repo, &branch).await.unwrap());
19248        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19249        assert_eq!(
19250            git::git(&repo, &["worktree", "list", "--porcelain"])
19251                .await
19252                .unwrap(),
19253            worktrees_before
19254        );
19255        let error = tokio::time::timeout(Duration::from_secs(3), &mut fold)
19256            .await
19257            .unwrap()
19258            .unwrap_err();
19259        assert!(error.is::<crate::repo_lock::ResourceBlocked>());
19260        drop(fold);
19261        assert!(candidate.exists() && reviewer.exists());
19262        assert!(!state.candidates[0].folded && state.enabled_worktree_config);
19263        assert_eq!(
19264            state.resource_block.as_ref().unwrap().kind,
19265            "repository coordination"
19266        );
19267        assert_eq!(std::fs::read(common.join("config")).unwrap(), config_before);
19268        drop(external);
19269        let mut fold = Box::pin(fold_run(&mut state, true, &home));
19270        let removed = tokio::time::timeout(Duration::from_secs(10), &mut fold)
19271            .await
19272            .unwrap()
19273            .unwrap();
19274        drop(fold);
19275        assert!(removed.contains(&branch));
19276        assert!(!root.exists());
19277        assert!(!git::branch_exists(&repo, &branch).await.unwrap());
19278        assert!(state.candidates[0].folded);
19279        assert!(!state.enabled_worktree_config);
19280        assert_ne!(std::fs::read(common.join("config")).unwrap(), config_before);
19281        let enabled = git::git_raw(
19282            &repo,
19283            &["config", "--local", "--get", "extensions.worktreeConfig"],
19284        )
19285        .await
19286        .unwrap();
19287        assert!(!enabled.ok(), "the last config reference was released");
19288    }
19289
19290    /// A branch handed to a later run is that run's (and its pull request's):
19291    /// folding the run that released it must not delete it.
19292    #[tokio::test]
19293    async fn fold_run_keeps_a_branch_that_was_handed_to_a_later_run() {
19294        let tmp = tempfile::tempdir().expect("tempdir");
19295        let repo = tmp.path().join("repo");
19296        std::fs::create_dir_all(&repo).unwrap();
19297        init_repo(&repo);
19298        let home = tmp.path().join("home");
19299
19300        let mut config = Config::default();
19301        config.graph.worktree_root = Some(tmp.path().join("wt"));
19302        let mut state = RunState::new(
19303            repo.clone(),
19304            "main".to_owned(),
19305            "deadbeef".to_owned(),
19306            "task".to_owned(),
19307            config,
19308        );
19309        // The worktree is already gone (released); the branch survives.
19310        git::git(&repo, &["branch", "magi/x/A", "main"])
19311            .await
19312            .expect("branch");
19313        state.candidates = vec![Candidate {
19314            index: 0,
19315            label: 'A',
19316            agent: "alpha".to_owned(),
19317            branch: "magi/x/A".to_owned(),
19318            worktree: state.worktree_root().join("cand-A"),
19319            summary: String::new(),
19320            stat: String::new(),
19321            files: 0,
19322            commits: 0,
19323            empty: false,
19324            failed: None,
19325            verified_noop: None,
19326            duration_ms: 0,
19327            folded: true,
19328        }];
19329        state.released_to = Some("20260901-000000-new1".to_owned());
19330        state.released_branches = vec!["magi/x/A".to_owned()];
19331
19332        fold_run(&mut state, true, &home).await.expect("fold_run");
19333        state.status = RunStatus::Merged;
19334        state.base_commit = git::rev_parse(&repo, "main").await.unwrap();
19335        state.merge = Some(crate::run::MergeOutcome {
19336            mode: MergeMode::Pr,
19337            ok: true,
19338            detail: "confirmed merged".into(),
19339            empty: false,
19340        });
19341        fold_run(&mut state, true, &home)
19342            .await
19343            .expect("merged fold");
19344
19345        assert!(
19346            git::branch_exists(&repo, "magi/x/A").await.unwrap(),
19347            "the handed-over branch survives a fold"
19348        );
19349    }
19350
19351    /// A winner with nothing ahead of the base is caught before `gh` is ever
19352    /// asked for a pull request, and the message carries what the task's
19353    /// references resolved to.
19354    #[tokio::test]
19355    async fn an_empty_winner_is_detected_before_a_pull_request_is_attempted() {
19356        let tmp = tempfile::tempdir().expect("tempdir");
19357        let repo = tmp.path().join("repo");
19358        std::fs::create_dir_all(&repo).unwrap();
19359        init_repo(&repo);
19360        let run = |args: &[&str]| {
19361            let out = std::process::Command::new("git")
19362                .quiet()
19363                .args(args)
19364                .current_dir(&repo)
19365                .output()
19366                .expect("spawn git");
19367            assert!(out.status.success(), "git {args:?}");
19368        };
19369        run(&["branch", "magi/x/A"]);
19370        run(&["checkout", "-q", "-b", "magi/x/B"]);
19371        std::fs::write(repo.join("f.txt"), "x\n").unwrap();
19372        run(&["add", "-A"]);
19373        run(&["commit", "-q", "-m", "work"]);
19374        run(&["checkout", "-q", "main"]);
19375
19376        // A pull request is compared against the remote's base, so the
19377        // fixture needs one. Before it exists nothing can be read, and the
19378        // answer must be "not empty".
19379        let probe = RunState::new(
19380            repo.clone(),
19381            "main".to_owned(),
19382            "deadbeef".to_owned(),
19383            "task".to_owned(),
19384            Config::default(),
19385        );
19386        assert!(!merge_is_empty(&repo, &probe, "magi/x/A", MergeMode::Pr).await);
19387        let bare = tmp.path().join("origin.git");
19388        let out = std::process::Command::new("git")
19389            .quiet()
19390            .args(["init", "-q", "--bare"])
19391            .arg(&bare)
19392            .output()
19393            .expect("spawn git");
19394        assert!(out.status.success(), "git init --bare");
19395        run(&["remote", "add", "origin", bare.to_str().unwrap()]);
19396        run(&["push", "-q", "origin", "main"]);
19397
19398        let mut state = RunState::new(
19399            repo.clone(),
19400            "main".to_owned(),
19401            "deadbeef".to_owned(),
19402            "task".to_owned(),
19403            Config::default(),
19404        );
19405        state.seeds = vec![refs::Seed {
19406            token: "magi/27b2/A".to_owned(),
19407            kind: refs::SeedKind::Unresolved,
19408            sha: String::new(),
19409            branch: true,
19410            detail: "no branch or commit named magi/27b2/A".to_owned(),
19411        }];
19412
19413        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Pr).await);
19414        assert!(merge_is_empty(&repo, &state, "magi/x/A", MergeMode::Local).await);
19415        assert!(!merge_is_empty(&repo, &state, "magi/x/B", MergeMode::Pr).await);
19416        let detail = empty_candidate_detail(&state, "main");
19417        assert!(detail.starts_with("empty candidate"), "{detail}");
19418        assert!(!detail.contains("magi/27b2/A"), "{detail}");
19419        assert!(detail.contains("--start-from"), "{detail}");
19420    }
19421
19422    /// `status == Ready` used to be read as "this is the harmless
19423    /// `MergeMode::None` no-op path, nothing to guard" (graph.rs, prior to
19424    /// this test). But `land` sets the very same status when a `MergeMode::Pr`
19425    /// run's PR was closed without merging — and reentering `merge` with
19426    /// `mode` still `Pr` does not know the difference, so it pushed and
19427    /// opened a second pull request. `mode == Local` reproduces the same
19428    /// blind spot without a network call: reentry must not attempt another
19429    /// git merge once this node has already recorded an outcome.
19430    #[tokio::test]
19431    async fn merge_does_not_reattempt_once_a_run_has_concluded() {
19432        let tmp = tempfile::tempdir().expect("tempdir");
19433        let repo = tmp.path().join("repo");
19434        std::fs::create_dir_all(&repo).unwrap();
19435        init_repo(&repo);
19436
19437        let mut config = Config::default();
19438        config.merge.mode = MergeMode::Local;
19439
19440        let mut state = RunState::new(
19441            repo.clone(),
19442            "main".to_owned(),
19443            "deadbeef".to_owned(),
19444            "task".to_owned(),
19445            config,
19446        );
19447        state.candidates = vec![Candidate {
19448            index: 0,
19449            label: 'A',
19450            agent: "alpha".to_owned(),
19451            branch: "does-not-exist".to_owned(),
19452            worktree: repo.clone(),
19453            summary: String::new(),
19454            stat: String::new(),
19455            files: 0,
19456            commits: 0,
19457            empty: false,
19458            failed: None,
19459            verified_noop: None,
19460            duration_ms: 0,
19461            folded: false,
19462        }];
19463        state.tally = Some(Tally {
19464            first_choice: BTreeMap::from([('A', 1)]),
19465            borda: BTreeMap::new(),
19466            winner: 'A',
19467            rankings: 1,
19468            unanimous_initial: true,
19469            deliberated: false,
19470            changed_votes: 0,
19471            unanimous_final: true,
19472            tie_break: None,
19473            judges: 0,
19474            present: 0,
19475            quorum: 0,
19476            met_quorum: true,
19477            uncontested: Some("only candidate A produced a change".to_owned()),
19478        });
19479        state.reviews = vec![ReviewRound {
19480            round: 1,
19481            head: "deadbeef".to_owned(),
19482            verified_head: None,
19483            verified_at: None,
19484            reviews: Vec::new(),
19485            e2e: Vec::new(),
19486            fix: None,
19487            blocking: 0,
19488            answered: 0,
19489            expected: 0,
19490            clean: true,
19491            verify_retried: false,
19492            e2e_deferred: false,
19493            e2e_defer_reason: None,
19494            progressed: false,
19495            vote_split: false,
19496            reconsideration: Vec::new(),
19497            verdict: None,
19498        }];
19499        state.gate = vec![CommandOutcome {
19500            command: "test".to_owned(),
19501            code: Some(0),
19502            output_tail: String::new(),
19503            duration_ms: 0,
19504            resource_blocked: false,
19505        }];
19506        state.gate_ran = true;
19507        // Reached its conclusion already — e.g. `land` closing the PR without
19508        // merging it, which (like the honest `MergeMode::None` path) leaves
19509        // `status` at `Ready`. The recorded outcome is what actually marks
19510        // this node done.
19511        state.status = RunStatus::Ready;
19512        state.merge = Some(MergeOutcome {
19513            mode: MergeMode::Local,
19514            ok: false,
19515            detail: "already concluded".to_owned(),
19516            empty: false,
19517        });
19518
19519        let mut runner = Runner {
19520            state,
19521            roles: ResolvedRoles {
19522                implementers: Vec::new(),
19523                judges: Vec::new(),
19524                reviewers: Vec::new(),
19525                fixer: None,
19526                conductor: conductor(),
19527                implementer_roster: Vec::new(),
19528                judge_roster: Vec::new(),
19529                reviewer_roster: Vec::new(),
19530            },
19531            sem: Arc::new(Semaphore::new(1)),
19532            pause: Pause::new(),
19533            interrupt: Pause::new(),
19534        };
19535
19536        runner.merge().await.expect("merge");
19537
19538        assert_eq!(
19539            runner.state.status,
19540            RunStatus::Ready,
19541            "a concluded run's status must not change on reentry"
19542        );
19543        assert_eq!(
19544            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
19545            Some("already concluded"),
19546            "merge must not run again once the node already recorded an outcome"
19547        );
19548    }
19549
19550    /// `gate` leaves `state.gate_ran` false both before it has ever run and
19551    /// when its last attempt was resource-blocked (the shared build cache
19552    /// could not be acquired or confirmed fresh in time - see
19553    /// `CommandOutcome::resource_blocked`'s own doc). Trusting the empty
19554    /// `Vec` this also leaves behind used to read as "nothing failed" and let
19555    /// a run merge a tree the gate never actually checked - exactly the case
19556    /// a contended cache produces on every retry until it clears. `merge`
19557    /// must refuse until `gate` has actually recorded an attempt.
19558    #[tokio::test]
19559    async fn merge_refuses_a_gate_that_has_not_actually_run() {
19560        let tmp = tempfile::tempdir().expect("tempdir");
19561        let repo = tmp.path().join("repo");
19562        std::fs::create_dir_all(&repo).unwrap();
19563        init_repo(&repo);
19564
19565        let mut config = Config::default();
19566        config.merge.mode = MergeMode::Local;
19567
19568        let mut state = RunState::new(
19569            repo.clone(),
19570            "main".to_owned(),
19571            "deadbeef".to_owned(),
19572            "task".to_owned(),
19573            config,
19574        );
19575        state.candidates = vec![Candidate {
19576            index: 0,
19577            label: 'A',
19578            agent: "alpha".to_owned(),
19579            branch: "does-not-exist".to_owned(),
19580            worktree: repo.clone(),
19581            summary: String::new(),
19582            stat: String::new(),
19583            files: 0,
19584            commits: 0,
19585            empty: false,
19586            failed: None,
19587            verified_noop: None,
19588            duration_ms: 0,
19589            folded: false,
19590        }];
19591        state.tally = Some(Tally {
19592            first_choice: BTreeMap::from([('A', 1)]),
19593            borda: BTreeMap::new(),
19594            winner: 'A',
19595            rankings: 1,
19596            unanimous_initial: true,
19597            deliberated: false,
19598            changed_votes: 0,
19599            unanimous_final: true,
19600            tie_break: None,
19601            judges: 0,
19602            present: 0,
19603            quorum: 0,
19604            met_quorum: true,
19605            uncontested: Some("only candidate A produced a change".to_owned()),
19606        });
19607        state.reviews = vec![ReviewRound {
19608            round: 1,
19609            head: "deadbeef".to_owned(),
19610            verified_head: None,
19611            verified_at: None,
19612            reviews: Vec::new(),
19613            e2e: Vec::new(),
19614            fix: None,
19615            blocking: 0,
19616            answered: 0,
19617            expected: 0,
19618            clean: true,
19619            verify_retried: false,
19620            e2e_deferred: false,
19621            e2e_defer_reason: None,
19622            progressed: false,
19623            vote_split: false,
19624            reconsideration: Vec::new(),
19625            verdict: None,
19626        }];
19627        // The point: `gate` has not recorded anything yet.
19628        state.gate = Vec::new();
19629        state.gate_ran = false;
19630        state.status = RunStatus::Gating;
19631
19632        let mut runner = Runner {
19633            state,
19634            roles: ResolvedRoles {
19635                implementers: Vec::new(),
19636                judges: Vec::new(),
19637                reviewers: Vec::new(),
19638                fixer: None,
19639                conductor: conductor(),
19640                implementer_roster: Vec::new(),
19641                judge_roster: Vec::new(),
19642                reviewer_roster: Vec::new(),
19643            },
19644            sem: Arc::new(Semaphore::new(1)),
19645            pause: Pause::new(),
19646            interrupt: Pause::new(),
19647        };
19648
19649        runner.merge().await.expect("merge");
19650
19651        assert!(
19652            runner.state.merge.is_none(),
19653            "an empty gate must never be read as a passing one: {:?}",
19654            runner.state.merge
19655        );
19656    }
19657
19658    /// The `shoka` repro this schema bump exists for: `verify.gate` has no
19659    /// commands configured and `merge.mode` is `none` (a review-only run).
19660    /// `gate` must still record a real attempt — zero commands, vacuously
19661    /// passed — rather than leaving `state.gate` empty in a way `merge`
19662    /// cannot tell apart from "never ran"; otherwise the run reaches
19663    /// `Gating` and can never leave it. See `RunState::gate_ran`'s own doc.
19664    #[tokio::test]
19665    async fn gate_and_merge_reach_ready_when_no_gate_commands_are_configured() {
19666        let tmp = tempfile::tempdir().expect("tempdir");
19667        let repo = tmp.path().join("repo");
19668        std::fs::create_dir_all(&repo).unwrap();
19669        init_repo(&repo);
19670
19671        // Default config: `verify.gate` empty, `merge.mode` is `none`.
19672        let config = Config::default();
19673
19674        let mut state = RunState::new(
19675            repo.clone(),
19676            "main".to_owned(),
19677            "deadbeef".to_owned(),
19678            "task".to_owned(),
19679            config,
19680        );
19681        state.candidates = vec![Candidate {
19682            index: 0,
19683            label: 'A',
19684            agent: "alpha".to_owned(),
19685            branch: "does-not-exist".to_owned(),
19686            worktree: repo.clone(),
19687            summary: String::new(),
19688            stat: String::new(),
19689            files: 0,
19690            commits: 0,
19691            empty: false,
19692            failed: None,
19693            verified_noop: None,
19694            duration_ms: 0,
19695            folded: false,
19696        }];
19697        state.tally = Some(Tally {
19698            first_choice: BTreeMap::from([('A', 1)]),
19699            borda: BTreeMap::new(),
19700            winner: 'A',
19701            rankings: 1,
19702            unanimous_initial: true,
19703            deliberated: false,
19704            changed_votes: 0,
19705            unanimous_final: true,
19706            tie_break: None,
19707            judges: 0,
19708            present: 0,
19709            quorum: 0,
19710            met_quorum: true,
19711            uncontested: Some("only candidate A produced a change".to_owned()),
19712        });
19713        state.reviews = vec![ReviewRound {
19714            round: 1,
19715            head: "deadbeef".to_owned(),
19716            verified_head: None,
19717            verified_at: None,
19718            reviews: Vec::new(),
19719            e2e: Vec::new(),
19720            fix: None,
19721            blocking: 0,
19722            answered: 0,
19723            expected: 0,
19724            clean: true,
19725            verify_retried: false,
19726            e2e_deferred: false,
19727            e2e_defer_reason: None,
19728            progressed: false,
19729            vote_split: false,
19730            reconsideration: Vec::new(),
19731            verdict: None,
19732        }];
19733
19734        let mut runner = Runner {
19735            state,
19736            roles: ResolvedRoles {
19737                implementers: Vec::new(),
19738                judges: Vec::new(),
19739                reviewers: Vec::new(),
19740                fixer: None,
19741                conductor: conductor(),
19742                implementer_roster: Vec::new(),
19743                judge_roster: Vec::new(),
19744                reviewer_roster: Vec::new(),
19745            },
19746            sem: Arc::new(Semaphore::new(1)),
19747            pause: Pause::new(),
19748            interrupt: Pause::new(),
19749        };
19750
19751        runner.gate().await.expect("gate");
19752        assert!(
19753            runner.state.gate_ran,
19754            "zero configured commands is still a real attempt, not an unrun gate"
19755        );
19756        assert!(runner.state.gate.is_empty());
19757        assert_eq!(runner.state.gate_status(), GateStatus::NotConfigured);
19758        assert_ne!(
19759            runner.state.status,
19760            RunStatus::Blocked,
19761            "a gate with nothing to check must not read as failed"
19762        );
19763
19764        runner.merge().await.expect("merge");
19765        assert_eq!(
19766            runner.state.status,
19767            RunStatus::Ready,
19768            "a clean review-only run with no gate commands must reach Ready, not stay stuck in Gating"
19769        );
19770    }
19771
19772    /// `Config::cache_dir` is derived from `verify.e2e` as well as
19773    /// `verify.gate` (so the e2e leg and the final gate never build against
19774    /// different directories). With zero `verify.gate` commands but a
19775    /// `CARGO_TARGET_DIR`-using `verify.e2e`, `gate` used to still queue for
19776    /// that lease before discovering it had nothing to run - so a repo with
19777    /// no gate commands could come back `resource_blocked` (and therefore
19778    /// still `gate_ran == false`) on nothing but an unrelated run holding the
19779    /// cache, exactly the contention this run's own zero commands could
19780    /// never have touched. `gate` must recognise there is nothing to check
19781    /// before it ever asks for the lease.
19782    #[tokio::test]
19783    async fn gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run() {
19784        crate::run::pin_test_home();
19785        let home = crate::run::home();
19786
19787        let tmp = tempfile::tempdir().expect("tempdir");
19788        let repo = tmp.path().join("repo");
19789        std::fs::create_dir_all(&repo).unwrap();
19790        init_repo(&repo);
19791        // Unique to this test, so holding its lease cannot collide with
19792        // another test sharing the same process-wide `home`.
19793        let cache_dir = tmp.path().join("target");
19794
19795        let mut config = Config::default();
19796        config.verify.e2e = vec![format!("CARGO_TARGET_DIR='{}' true", cache_dir.display())];
19797        // `verify.gate` stays empty (the default). Any accidental cache
19798        // acquisition must report contention immediately, regardless of how
19799        // long unrelated Git subprocesses take under parallel test load.
19800        config.graph.timeout_verify = Some(2);
19801        config.graph.timeout_resource_wait = 0;
19802
19803        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
19804        let _held = match crate::cache::try_acquire(&home, &cache_dir, &other)
19805            .expect("no io error acquiring directly")
19806        {
19807            crate::cache::AcquireOutcome::Acquired(g) => g,
19808            crate::cache::AcquireOutcome::Busy(b) => {
19809                panic!("expected the direct acquire to win the lease first: {b:?}")
19810            }
19811        };
19812
19813        let mut state = RunState::new(
19814            repo.clone(),
19815            "main".to_owned(),
19816            "deadbeef".to_owned(),
19817            "task".to_owned(),
19818            config,
19819        );
19820        state.candidates = vec![Candidate {
19821            index: 0,
19822            label: 'A',
19823            agent: "alpha".to_owned(),
19824            branch: "does-not-exist".to_owned(),
19825            worktree: repo.clone(),
19826            summary: String::new(),
19827            stat: String::new(),
19828            files: 0,
19829            commits: 0,
19830            empty: false,
19831            failed: None,
19832            verified_noop: None,
19833            duration_ms: 0,
19834            folded: false,
19835        }];
19836        state.tally = Some(Tally {
19837            first_choice: BTreeMap::from([('A', 1)]),
19838            borda: BTreeMap::new(),
19839            winner: 'A',
19840            rankings: 1,
19841            unanimous_initial: true,
19842            deliberated: false,
19843            changed_votes: 0,
19844            unanimous_final: true,
19845            tie_break: None,
19846            judges: 0,
19847            present: 0,
19848            quorum: 0,
19849            met_quorum: true,
19850            uncontested: Some("only candidate A produced a change".to_owned()),
19851        });
19852        state.reviews = vec![ReviewRound {
19853            round: 1,
19854            head: "deadbeef".to_owned(),
19855            verified_head: None,
19856            verified_at: None,
19857            reviews: Vec::new(),
19858            e2e: Vec::new(),
19859            fix: None,
19860            blocking: 0,
19861            answered: 0,
19862            expected: 0,
19863            clean: true,
19864            verify_retried: false,
19865            e2e_deferred: false,
19866            e2e_defer_reason: None,
19867            progressed: false,
19868            vote_split: false,
19869            reconsideration: Vec::new(),
19870            verdict: None,
19871        }];
19872
19873        state.reviews[0].verified_head = Some(git::rev_parse(&repo, "HEAD").await.unwrap());
19874        state.reviews[0].e2e = vec![CommandOutcome {
19875            command: state.config.verify.e2e[0].clone(),
19876            code: Some(0),
19877            output_tail: String::new(),
19878            duration_ms: 0,
19879            resource_blocked: false,
19880        }];
19881        let mut runner = Runner {
19882            state,
19883            roles: ResolvedRoles {
19884                implementers: Vec::new(),
19885                judges: Vec::new(),
19886                reviewers: Vec::new(),
19887                fixer: None,
19888                conductor: conductor(),
19889                implementer_roster: Vec::new(),
19890                judge_roster: Vec::new(),
19891                reviewer_roster: Vec::new(),
19892            },
19893            sem: Arc::new(Semaphore::new(1)),
19894            pause: Pause::new(),
19895            interrupt: Pause::new(),
19896        };
19897
19898        runner.gate().await.expect("gate");
19899        assert!(runner.state.resource_block.is_none());
19900        assert!(
19901            !runner
19902                .state
19903                .events
19904                .iter()
19905                .any(|event| { event.message.contains("waiting for the shared build cache") }),
19906            "a gate with nothing to run must never request the held cache lease"
19907        );
19908        assert!(
19909            runner.state.gate_ran,
19910            "zero commands is still a real, immediate attempt"
19911        );
19912        assert!(runner.state.gate.is_empty());
19913        assert_ne!(
19914            runner.state.status,
19915            RunStatus::Blocked,
19916            "must not read as resource-blocked on a lease it never asked for"
19917        );
19918    }
19919
19920    /// A separate process carries a live worker's environment throughout all
19921    /// verification entry points; the parallel harness never changes its env.
19922    #[cfg(unix)]
19923    #[tokio::test]
19924    async fn verification_is_standalone_inside_a_live_worker() {
19925        const CHILD: &str = "MAGI_VERIFY_WORKER_TEST";
19926        if std::env::var_os(CHILD).is_none() {
19927            let home = tempfile::tempdir().unwrap();
19928            let generation = crate::worker::pin_current(home.path()).unwrap();
19929            let output = std::process::Command::new(std::env::current_exe().unwrap())
19930                .args([
19931                    "--exact",
19932                    "graph::tests::verification_is_standalone_inside_a_live_worker",
19933                    "--nocapture",
19934                ])
19935                .env(CHILD, "1")
19936                .env("MAGI_HOME", home.path())
19937                .env("MAGI_WORKER_HOME", home.path())
19938                .env("MAGI_WORKER_ID", "abcdef")
19939                .env("MAGI_WORKER_GENERATION", &generation.id)
19940                .env("MAGI_WORKER_EXE", &generation.executable)
19941                .env("MAGI_RUN", "verification-parent")
19942                .env("MAGI_NODE", "gate")
19943                .env("MAGI_SEAT", "worker-fixture")
19944                .env("MAGI_PROCESS_OWNER", "fixture-owner")
19945                .env("MAGI_NO_AUTOUPDATE", "0")
19946                .env("VERIFY_EXPECT_PATH", std::env::var_os("PATH").unwrap())
19947                .env("VERIFY_AUTH_TOKEN", "fixture-auth")
19948                .env(
19949                    "VERIFY_EXPECT_TOOLCHAIN",
19950                    std::env::var_os("RUSTUP_TOOLCHAIN").unwrap_or_default(),
19951                )
19952                .env(
19953                    "VERIFY_EXPECT_CACHE",
19954                    std::env::var_os("CARGO_TARGET_DIR").unwrap_or_default(),
19955                )
19956                .output()
19957                .unwrap();
19958            assert!(
19959                output.status.success()
19960                    && String::from_utf8_lossy(&output.stdout).contains("1 passed"),
19961                "{}\n{}",
19962                String::from_utf8_lossy(&output.stdout),
19963                String::from_utf8_lossy(&output.stderr)
19964            );
19965            return;
19966        }
19967        let home = crate::run::home();
19968        let directory = home.join("workers/jobs/abcdef");
19969        std::fs::create_dir_all(&directory).unwrap();
19970        let job = crate::worker::Job {
19971            id: "abcdef".into(),
19972            sequence: 1,
19973            kind: "task".into(),
19974            key: "task:verification".into(),
19975            payload: serde_json::json!({}),
19976            generation: std::env::var("MAGI_WORKER_GENERATION").unwrap(),
19977            version: Some(env!("CARGO_PKG_VERSION").into()),
19978            status: crate::worker::Status::Running,
19979            result: None,
19980            pid: Some(std::process::id()),
19981            created_at: 0,
19982            finished_at: None,
19983        };
19984        std::fs::write(
19985            directory.join("job.json"),
19986            serde_json::to_vec(&job).unwrap(),
19987        )
19988        .unwrap();
19989        let _lease = crate::worker::execution_lease_for_test(&directory.join("live.lock"));
19990        let assert_worker = || {
19991            assert_eq!(
19992                crate::worker::current_job(&home).unwrap().unwrap().id,
19993                job.id
19994            );
19995            assert_eq!(crate::worker::active(&home).unwrap().len(), 1);
19996            assert_eq!(
19997                crate::worker::helper_executable(&home).unwrap(),
19998                PathBuf::from(std::env::var_os("MAGI_WORKER_EXE").unwrap())
19999            );
20000            assert!(
20001                crate::updater::Checker::new(&crate::config::Update {
20002                    mode: crate::config::UpdateMode::Notify,
20003                    interval: None,
20004                    auto_restart: false,
20005                })
20006                .is_none(),
20007                "the driver remains pinned, even while tests are standalone"
20008            );
20009        };
20010        assert_worker();
20011        let repo = home.join("repo");
20012        std::fs::create_dir_all(&repo).unwrap();
20013        init_repo(&repo);
20014        // Keep storage, seat/control context, containment and tool lookup.
20015        // No inherited values (or credentials) are printed on success.
20016        let probe = r#"test -z "${MAGI_WORKER_ID+x}" &&
20017            test -z "${MAGI_WORKER_GENERATION+x}" &&
20018            test -z "${MAGI_WORKER_HOME+x}" &&
20019            test -z "${MAGI_WORKER_EXE+x}" &&
20020            test "$MAGI_HOME/repo" -ef . &&
20021            test "$MAGI_RUN" = verification-parent &&
20022            test "$MAGI_NODE" = gate &&
20023            test "$MAGI_SEAT" = worker-fixture &&
20024            test -n "$MAGI_PROCESS_OWNER" &&
20025            test "$PATH" = "$VERIFY_EXPECT_PATH" &&
20026            test "$VERIFY_AUTH_TOKEN" = fixture-auth &&
20027            test "$RUSTUP_TOOLCHAIN" = "$VERIFY_EXPECT_TOOLCHAIN" &&
20028            test "$CARGO_TARGET_DIR" = "$VERIFY_EXPECT_CACHE""#;
20029        // Negative control: an ordinary child inherits the contamination.
20030        assert!(
20031            !std::process::Command::new("sh")
20032                .args(["-c", probe])
20033                .current_dir(&repo)
20034                .status()
20035                .unwrap()
20036                .success()
20037        );
20038        let unrelated_home = tempfile::tempdir().unwrap();
20039        assert!(
20040            crate::worker::current_job(unrelated_home.path()).is_err(),
20041            "an inherited worker id cannot be resolved in a unit fixture's home"
20042        );
20043        let mut runner = runner_at(RunStatus::Gating);
20044        runner.state.repo = repo.clone();
20045        runner.state.config.disk.min_free_bytes = 0;
20046        runner.state.config.coordination = crate::config::test_coordination();
20047        std::fs::write(repo.join(".git/updater.contract"), "test-protocol-v1\n").unwrap();
20048        runner.state.config.verify.e2e = vec![format!("{probe} && printf 'e2e\\n' >> phases")];
20049        runner.state.config.verify.pre_gate = vec![format!("{probe} && printf 'pre\\n' >> phases")];
20050        runner.state.config.verify.gate = vec![format!("{probe} && printf 'gate\\n' >> phases")];
20051        let winner = Candidate {
20052            index: 0,
20053            label: 'A',
20054            agent: "fixture".into(),
20055            branch: "main".into(),
20056            worktree: repo.clone(),
20057            summary: String::new(),
20058            stat: String::new(),
20059            files: 0,
20060            commits: 0,
20061            empty: false,
20062            failed: None,
20063            verified_noop: None,
20064            duration_ms: 0,
20065            folded: false,
20066        };
20067        let (e2e, retried, pids) = run_e2e_with_retry(
20068            &mut runner.state,
20069            &["sh".into(), "-c".into()],
20070            &[format!("{probe} && printf initial")],
20071            &repo,
20072            Duration::from_secs(10),
20073            "initial",
20074        )
20075        .await;
20076        assert!(!retried && pids.is_empty());
20077        assert_eq!(e2e[0].code, Some(0), "{e2e:?}");
20078        assert_eq!(e2e[0].output_tail, "initial");
20079        let gate = runner.run_gate(&winner).await.unwrap();
20080        assert!(gate.iter().all(CommandOutcome::ok), "{gate:?}");
20081        // An interrupted repair must recheck e2e from saved state. Landing
20082        // reuses that state and must run both pre-gate and gate again.
20083        runner.state.gate_repair_e2e = Some(crate::run::GateRepairE2e {
20084            head: git::rev_parse(&repo, "HEAD").await.unwrap(),
20085            passed: false,
20086            outcomes: Vec::new(),
20087        });
20088        runner.state.save().unwrap();
20089        let mut resumed = RunState::load(&runner.state.id).unwrap();
20090        let wait = crate::repo_lock::WaitPolicy::default();
20091        assert!(matches!(
20092            check_gate_repair_e2e(&mut resumed, &winner, &wait)
20093                .await
20094                .unwrap(),
20095            GateFix::Retry
20096        ));
20097        let saved = RunState::load(&resumed.id).unwrap();
20098        let repair = saved.gate_repair_e2e.unwrap();
20099        assert!(repair.passed);
20100        assert_eq!(repair.outcomes.len(), 1);
20101        assert!(repair.outcomes[0].ok());
20102        let landing = run_landing_gate(&mut resumed, &winner, &wait)
20103            .await
20104            .unwrap();
20105        assert_eq!(landing.len(), 1);
20106        assert!(landing[0].ok(), "{landing:?}");
20107        assert_eq!(
20108            std::fs::read_to_string(repo.join("phases")).unwrap(),
20109            "pre\ngate\ne2e\npre\ngate\n"
20110        );
20111        assert_worker();
20112    }
20113
20114    #[cfg(unix)]
20115    #[tokio::test]
20116    async fn verification_keeps_the_panic_and_the_full_output() {
20117        crate::run::pin_test_home();
20118        let tmp = tempfile::tempdir().unwrap();
20119        let mut config = Config::default();
20120        config.disk.min_free_bytes = 0;
20121        let mut state = RunState::new(
20122            tmp.path().to_owned(),
20123            "main".into(),
20124            "base".into(),
20125            "verification evidence".into(),
20126            config,
20127        );
20128        let script = "echo \"thread 'x' panicked at t.rs:1:1:\"; echo 'boom assertion'; \
20129                      i=0; while [ $i -lt 20000 ]; do echo \"  $i: 0xabc - <a::b>\"; \
20130                      echo '             at /x/y.rs:1:1'; i=$((i+1)); done; exit 101";
20131        let mut names = Vec::new();
20132        for _ in 0..2 {
20133            let (outcomes, _) = run_commands(
20134                &mut state,
20135                "gate",
20136                "gate",
20137                0,
20138                &["sh".into(), "-c".into()],
20139                &[script.into()],
20140                tmp.path(),
20141                Duration::from_secs(60),
20142            )
20143            .await;
20144            let tail = &outcomes[0].output_tail;
20145            assert!(tail.contains("panicked at t.rs:1:1"), "{tail}");
20146            assert!(tail.contains("boom assertion"));
20147            assert!(tail.len() <= crate::run::VERIFY_SUMMARY + 100);
20148            let line = tail.lines().next().unwrap();
20149            names.push(line.to_owned());
20150        }
20151        assert_ne!(names[0], names[1], "the second run must not overwrite");
20152        let full = crate::run::read_artifact(&state, "gate-1.out").unwrap();
20153        assert!(full.len() > 500_000 && full.contains("boom assertion"));
20154        assert!(crate::run::read_artifact(&state, "gate-2.out").is_some());
20155
20156        // Storage failures are evidence too; preserve the command's verdict.
20157        std::fs::remove_dir_all(state.dir().join("artifacts")).unwrap();
20158        std::fs::write(state.dir().join("artifacts"), "blocked").unwrap();
20159        let (outcomes, _) = run_commands(
20160            &mut state,
20161            "gate",
20162            "gate",
20163            0,
20164            &["sh".into(), "-c".into()],
20165            &["echo failure-output; exit 7".into()],
20166            tmp.path(),
20167            Duration::from_secs(10),
20168        )
20169        .await;
20170        assert_eq!(outcomes[0].code, Some(7));
20171        assert!(state.events.iter().any(|e| {
20172            e.node == "gate" && e.message.contains("could not save the full gate output:")
20173        }));
20174    }
20175
20176    #[cfg(unix)]
20177    #[tokio::test]
20178    async fn verification_commands_distinguish_timeouts_signals_and_exit_codes() {
20179        crate::run::pin_test_home();
20180        let tmp = tempfile::tempdir().unwrap();
20181        let mut config = Config::default();
20182        config.disk.min_free_bytes = 0;
20183        let mut state = RunState::new(
20184            tmp.path().to_owned(),
20185            "main".into(),
20186            "base".into(),
20187            "verification evidence".into(),
20188            config,
20189        );
20190        let (outcomes, pids) = run_commands(
20191            &mut state,
20192            "gate",
20193            "gate",
20194            1,
20195            &["sh".into(), "-c".into()],
20196            &[
20197                "printf before-timeout; exec sleep 30".into(),
20198                "kill -TERM $$".into(),
20199                "exit 7".into(),
20200                "printf success".into(),
20201            ],
20202            tmp.path(),
20203            Duration::from_secs(1),
20204        )
20205        .await;
20206        wait_for_timed_out_children_to_die(&pids).await;
20207        assert_eq!(
20208            pids.len(),
20209            1,
20210            "unleased timeout retains its death-check PID"
20211        );
20212        assert_eq!(outcomes.len(), 4);
20213        assert_eq!(outcomes[0].code, None);
20214        assert!(outcomes[0].output_tail.contains("before-timeout"));
20215        assert!(outcomes[0].output_tail.contains("timed out after 1s"));
20216        assert_eq!(outcomes[1].code, None);
20217        assert!(outcomes[1].output_tail.contains("command terminated:"));
20218        assert!(!outcomes[1].output_tail.contains("timed out"));
20219        assert_eq!(outcomes[2].code, Some(7));
20220        assert!(!outcomes[2].output_tail.contains("timed out"));
20221        assert_eq!(outcomes[3].code, Some(0));
20222        assert_eq!(outcomes[3].output_tail, "success");
20223        assert!(outcomes.iter().all(|outcome| !outcome.resource_blocked));
20224    }
20225
20226    /// The addendum's second gap: a `verify.gate` command running for real
20227    /// wall-clock time had nothing at all to show for it in `active` before
20228    /// `run_commands` learned to record it — a run could sit in `Gating` for
20229    /// minutes with `magi show` and `GET /api/runs/{id}` both silent about
20230    /// what was actually happening. Proven with a genuinely still-running
20231    /// command, not just a before/after check on the final state: a poller
20232    /// task reads the same `run.json` `gate()` is writing, the same way the
20233    /// phone or `magi show` would, while the shell command is still blocked
20234    /// on its own release marker.
20235    #[tokio::test]
20236    async fn gate_records_a_running_task_entry_while_its_command_is_still_in_flight() {
20237        crate::run::pin_test_home();
20238
20239        let tmp = tempfile::tempdir().expect("tempdir");
20240        let repo = tmp.path().join("repo");
20241        std::fs::create_dir_all(&repo).unwrap();
20242        init_repo(&repo);
20243
20244        let mut config = Config::default();
20245        config.disk.min_free_bytes = 0;
20246        config.verify.gate = vec![
20247            "printf started > started.marker; i=0; while [ ! -f release.marker ] && \
20248             [ \"$i\" -lt 100 ]; do i=$((i+1)); sleep 0.05; done"
20249                .to_owned(),
20250        ];
20251
20252        let mut state = RunState::new(
20253            repo.clone(),
20254            "main".to_owned(),
20255            "deadbeef".to_owned(),
20256            "task".to_owned(),
20257            config,
20258        );
20259        let run_id = state.id.clone();
20260        state.candidates = vec![Candidate {
20261            index: 0,
20262            label: 'A',
20263            agent: "alpha".to_owned(),
20264            branch: "does-not-exist".to_owned(),
20265            worktree: repo.clone(),
20266            summary: String::new(),
20267            stat: String::new(),
20268            files: 0,
20269            commits: 0,
20270            empty: false,
20271            failed: None,
20272            verified_noop: None,
20273            duration_ms: 0,
20274            folded: false,
20275        }];
20276        state.tally = Some(Tally {
20277            first_choice: BTreeMap::from([('A', 1)]),
20278            borda: BTreeMap::new(),
20279            winner: 'A',
20280            rankings: 1,
20281            unanimous_initial: true,
20282            deliberated: false,
20283            changed_votes: 0,
20284            unanimous_final: true,
20285            tie_break: None,
20286            judges: 0,
20287            present: 0,
20288            quorum: 0,
20289            met_quorum: true,
20290            uncontested: Some("only candidate A produced a change".to_owned()),
20291        });
20292        state.reviews = vec![ReviewRound {
20293            round: 1,
20294            head: "deadbeef".to_owned(),
20295            verified_head: None,
20296            verified_at: None,
20297            reviews: Vec::new(),
20298            e2e: Vec::new(),
20299            fix: None,
20300            blocking: 0,
20301            answered: 0,
20302            expected: 0,
20303            clean: true,
20304            verify_retried: false,
20305            e2e_deferred: false,
20306            e2e_defer_reason: None,
20307            progressed: false,
20308            vote_split: false,
20309            reconsideration: Vec::new(),
20310            verdict: None,
20311        }];
20312
20313        let mut runner = Runner {
20314            state,
20315            roles: ResolvedRoles {
20316                implementers: Vec::new(),
20317                judges: Vec::new(),
20318                reviewers: Vec::new(),
20319                fixer: None,
20320                conductor: conductor(),
20321                implementer_roster: Vec::new(),
20322                judge_roster: Vec::new(),
20323                reviewer_roster: Vec::new(),
20324            },
20325            sem: Arc::new(Semaphore::new(1)),
20326            pause: Pause::new(),
20327            interrupt: Pause::new(),
20328        };
20329
20330        let started_marker = repo.join("started.marker");
20331        let release_marker = repo.join("release.marker");
20332        let poller = tokio::spawn(async move {
20333            // Bounded so a regression that never records the task entry
20334            // fails this test in seconds instead of hanging the suite —
20335            // the same shape `a_park_requested_while_a_seat_is_mid_call_
20336            // does_not_cut_it_short` uses for the same reason.
20337            for _ in 0..100 {
20338                if started_marker.exists()
20339                    && let Ok(s) = crate::run::RunState::load(&run_id)
20340                    && let Some(a) = s.active.get("gate")
20341                {
20342                    std::fs::write(&release_marker, b"go").expect("release marker");
20343                    return Some(a.clone());
20344                }
20345                tokio::time::sleep(Duration::from_millis(50)).await;
20346            }
20347            None
20348        });
20349
20350        runner.gate().await.expect("gate");
20351        let captured = poller.await.expect("poller task");
20352        let captured = captured.expect(
20353            "the poller never saw a `gate` task entry in run.json while the command was \
20354             still blocked on its own release marker",
20355        );
20356
20357        assert_eq!(captured.task.as_deref(), Some("gate"));
20358        assert_eq!(captured.node, "gate");
20359        assert_eq!(captured.index, Some(1));
20360        assert_eq!(captured.total, Some(1));
20361        assert!(
20362            captured
20363                .command
20364                .as_deref()
20365                .is_some_and(|c| c.contains("started.marker")),
20366            "{captured:?}"
20367        );
20368
20369        assert!(
20370            runner.state.active.is_empty(),
20371            "the entry must be cleared once the command actually finished: {:?}",
20372            runner.state.active
20373        );
20374        assert!(runner.state.gate_ran);
20375        assert!(runner.state.gate.iter().all(CommandOutcome::ok));
20376    }
20377
20378    /// The hand-off over a blocking finding a reviewer rejected on leaves a
20379    /// record for `land`; one with only a Minor, or no reject, leaves none.
20380    #[tokio::test]
20381    async fn stop_reviewing_records_a_contested_hand_off_only_for_major_plus_reject() {
20382        use crate::verdict::{Finding, ReviewVote, Severity};
20383        crate::run::pin_test_home();
20384        let tmp = tempfile::tempdir().expect("tempdir");
20385        let repo = tmp.path().join("repo");
20386        std::fs::create_dir_all(&repo).unwrap();
20387        init_repo(&repo);
20388
20389        for (severity, vote, expect) in [
20390            (Severity::Major, ReviewVote::Reject, true),
20391            (Severity::Minor, ReviewVote::Reject, false),
20392            (Severity::Major, ReviewVote::Approve, false),
20393        ] {
20394            let mut round = review_round(false, 1, 1, 1, false, true);
20395            round.reviews = vec![ReviewRecord {
20396                reviewer: 1,
20397                agent: "alpha".to_owned(),
20398                summary: String::new(),
20399                findings: vec![Finding {
20400                    id: "R1-1-1".to_owned(),
20401                    severity,
20402                    file: None,
20403                    line: None,
20404                    title: "t".to_owned(),
20405                    detail: String::new(),
20406                }],
20407                vote: Some(vote),
20408                failed: None,
20409                duration_ms: 0,
20410                attempts: 0,
20411            }];
20412            let mut state = RunState::new(
20413                repo.clone(),
20414                "main".to_owned(),
20415                "deadbeef".to_owned(),
20416                "task".to_owned(),
20417                Config::default(),
20418            );
20419            state.reviews = vec![round];
20420            let mut runner = Runner {
20421                state,
20422                roles: ResolvedRoles {
20423                    implementers: Vec::new(),
20424                    judges: Vec::new(),
20425                    reviewers: Vec::new(),
20426                    fixer: None,
20427                    conductor: conductor(),
20428                    implementer_roster: Vec::new(),
20429                    judge_roster: Vec::new(),
20430                    reviewer_roster: Vec::new(),
20431                },
20432                sem: Arc::new(Semaphore::new(1)),
20433                pause: Pause::new(),
20434                interrupt: Pause::new(),
20435            };
20436            let shell = runner.state.config.shell();
20437            runner
20438                .stop_reviewing("round budget spent", &shell, &repo)
20439                .await
20440                .expect("stop_reviewing");
20441            assert_eq!(runner.state.status, RunStatus::Gating);
20442            assert_eq!(
20443                runner.state.contested_handoff.is_some(),
20444                expect,
20445                "{severity:?} + {vote:?}"
20446            );
20447        }
20448    }
20449
20450    /// The shape the incident this whole fix responds to actually had: the
20451    /// round budget spent, the last round's own e2e blocked on the shared
20452    /// build cache (held here by a live pid — this test process — exactly
20453    /// `cache`'s own unit tests' pattern for "another owner, still alive"
20454    /// without forking a process). `stop_reviewing` must retry it — not
20455    /// silently leave the round looking untouched (the catch-up-only half of
20456    /// the bug), and not read the contention as a red `e2e` and block the
20457    /// run on it (the other half). Called directly, the same way
20458    /// `gate_never_asks_for_the_cache_lease_when_it_has_no_commands_to_run`
20459    /// above exercises `gate`, so this never needs a real cargo build to
20460    /// reach: the lease is never released, so `with_cache_lease` never gets
20461    /// past acquiring it into anything that would need a real workspace.
20462    #[tokio::test]
20463    async fn stop_reviewing_retries_a_resource_blocked_e2e_instead_of_reading_it_as_red() {
20464        crate::run::pin_test_home();
20465        let home = crate::run::home();
20466
20467        let tmp = tempfile::tempdir().expect("tempdir");
20468        let repo = tmp.path().join("repo");
20469        std::fs::create_dir_all(&repo).unwrap();
20470        init_repo(&repo);
20471        let head = crate::git::rev_parse(&repo, "HEAD")
20472            .await
20473            .expect("rev-parse");
20474        // Unique to this test, so holding its lease cannot collide with
20475        // another test sharing the same process-wide `home`.
20476        let cache_dir = tmp.path().join("target");
20477
20478        let mut config = Config::default();
20479        config.verify.e2e = vec![format!(
20480            "CARGO_TARGET_DIR='{}' test -f README.md",
20481            cache_dir.display()
20482        )];
20483        config.graph.review_rounds = 1;
20484        // Bounded so a regression that does start waiting fails the test in
20485        // seconds, not hangs it.
20486        config.graph.timeout_verify = Some(2);
20487        config.graph.timeout_resource_wait = 2;
20488
20489        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20490        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20491            .expect("no io error acquiring directly")
20492        {
20493            crate::cache::AcquireOutcome::Acquired(g) => g,
20494            crate::cache::AcquireOutcome::Busy(b) => {
20495                panic!("expected the direct acquire to win the lease first: {b:?}")
20496            }
20497        };
20498
20499        let mut state = RunState::new(
20500            repo.clone(),
20501            "main".to_owned(),
20502            head.clone(),
20503            "task".to_owned(),
20504            config,
20505        );
20506        state.candidates = vec![Candidate {
20507            index: 0,
20508            label: 'A',
20509            agent: "alpha".to_owned(),
20510            branch: "does-not-exist".to_owned(),
20511            worktree: repo.clone(),
20512            summary: String::new(),
20513            stat: String::new(),
20514            files: 0,
20515            commits: 0,
20516            empty: false,
20517            failed: None,
20518            verified_noop: None,
20519            duration_ms: 0,
20520            folded: false,
20521        }];
20522        state.tally = Some(Tally {
20523            first_choice: BTreeMap::from([('A', 1)]),
20524            borda: BTreeMap::new(),
20525            winner: 'A',
20526            rankings: 1,
20527            unanimous_initial: true,
20528            deliberated: false,
20529            changed_votes: 0,
20530            unanimous_final: true,
20531            tie_break: None,
20532            judges: 0,
20533            present: 0,
20534            quorum: 0,
20535            met_quorum: true,
20536            uncontested: Some("only candidate A produced a change".to_owned()),
20537        });
20538        // The round budget's last round, deferred: `needs_catchup_run`'s
20539        // other trigger. `stop_reviewing`'s retry machinery must treat this
20540        // exactly like a resource-blocked attempt once it actually runs.
20541        state.reviews = vec![ReviewRound {
20542            round: 1,
20543            head: head.clone(),
20544            verified_head: None,
20545            verified_at: None,
20546            reviews: Vec::new(),
20547            e2e: Vec::new(),
20548            fix: None,
20549            blocking: 1,
20550            answered: 1,
20551            expected: 1,
20552            clean: false,
20553            verify_retried: false,
20554            e2e_deferred: true,
20555            e2e_defer_reason: Some("1 blocking finding(s) already required a fix".to_owned()),
20556            progressed: false,
20557            vote_split: false,
20558            reconsideration: Vec::new(),
20559            verdict: None,
20560        }];
20561
20562        let mut runner = Runner {
20563            state,
20564            roles: ResolvedRoles {
20565                implementers: Vec::new(),
20566                judges: Vec::new(),
20567                reviewers: Vec::new(),
20568                fixer: None,
20569                conductor: conductor(),
20570                implementer_roster: Vec::new(),
20571                judge_roster: Vec::new(),
20572                reviewer_roster: Vec::new(),
20573            },
20574            sem: Arc::new(Semaphore::new(1)),
20575            pause: Pause::new(),
20576            interrupt: Pause::new(),
20577        };
20578
20579        let shell = runner.state.config.shell();
20580        runner
20581            .stop_reviewing("round budget spent", &shell, &repo)
20582            .await
20583            .expect("stop_reviewing");
20584
20585        let last = runner.state.reviews.last().expect("round record");
20586        assert_eq!(
20587            last.e2e_status(),
20588            E2eStatus::ResourceBlocked,
20589            "the shared cache is still held; the attempt must read as blocked, not deferred or \
20590             failed: {last:?}"
20591        );
20592        assert_eq!(
20593            last.verified_head.as_deref(),
20594            Some(head.as_str()),
20595            "which commit this attempt targeted is known even though nothing finished checking \
20596             it"
20597        );
20598        let first_attempt_at = last
20599            .verified_at
20600            .expect("when this attempt ran is known too");
20601        assert_ne!(
20602            runner.state.status,
20603            RunStatus::Blocked,
20604            "contention is evidence about the machine, not the patch — it must not settle the \
20605             run as blocked: {:?}",
20606            runner.state.status
20607        );
20608        assert!(
20609            !runner
20610                .state
20611                .events
20612                .iter()
20613                .any(|e| e.node == "review" && e.message.contains("e2e failed")),
20614            "a resource-blocked attempt must never be logged as a failed e2e: {:?}",
20615            runner.state.events
20616        );
20617
20618        // The cache is still held: a later reentry must retry the same
20619        // round's verification again — not leave it looking exactly as
20620        // untouched as the first blocked attempt, which is indistinguishable
20621        // from never having tried again at all.
20622        runner
20623            .stop_reviewing("round budget spent", &shell, &repo)
20624            .await
20625            .expect("stop_reviewing retry");
20626        assert_eq!(
20627            runner.state.reviews.len(),
20628            1,
20629            "no new round was started: {:?}",
20630            runner.state.reviews
20631        );
20632        let last = runner.state.reviews.last().expect("round record");
20633        assert_eq!(last.e2e_status(), E2eStatus::ResourceBlocked, "{last:?}");
20634        assert!(
20635            last.verified_at.expect("still known") > first_attempt_at,
20636            "a second reentry must be a fresh attempt, not a stale copy of the first"
20637        );
20638        assert_ne!(runner.state.status, RunStatus::Blocked);
20639
20640        held.release();
20641    }
20642
20643    #[cfg(unix)]
20644    #[tokio::test]
20645    async fn catchup_compile_failure_after_clean_base_sync_uses_remaining_review_rounds() {
20646        ask_test_home();
20647        for (max_rounds, reentry) in [(2, false), (2, true), (1, false)] {
20648            let tmp = tempfile::tempdir().unwrap();
20649            let repo = tmp.path().join("repo");
20650            std::fs::create_dir(&repo).unwrap();
20651            init_repo(&repo);
20652            let run_git = |args: &[&str]| {
20653                let out = std::process::Command::new("git")
20654                    .args(args)
20655                    .current_dir(&repo)
20656                    .output()
20657                    .unwrap();
20658                assert!(
20659                    out.status.success(),
20660                    "{args:?}: {}",
20661                    String::from_utf8_lossy(&out.stderr)
20662                );
20663            };
20664            let origin = tmp.path().join("origin.git");
20665            run_git(&["init", "--bare", origin.to_str().unwrap()]);
20666            run_git(&["remote", "add", "origin", origin.to_str().unwrap()]);
20667            std::fs::write(repo.join("spec.rs"), "struct AgentSpec {}\n").unwrap();
20668            run_git(&["add", "spec.rs"]);
20669            run_git(&["commit", "-m", "initial spec"]);
20670            run_git(&["push", "origin", "main"]);
20671            let base = git::rev_parse(&repo, "HEAD").await.unwrap();
20672            run_git(&["checkout", "-b", "fix/review"]);
20673            std::fs::write(
20674                repo.join("check.rs"),
20675                "include!(\"spec.rs\"); fn main() { let _ = AgentSpec {}; }\n",
20676            )
20677            .unwrap();
20678            run_git(&["add", "check.rs"]);
20679            run_git(&["commit", "-m", "use spec"]);
20680            let reviewed = git::rev_parse(&repo, "HEAD").await.unwrap();
20681            run_git(&["checkout", "main"]);
20682            std::fs::write(
20683                repo.join("spec.rs"),
20684                "struct AgentSpec { max_turn_secs: u64 }\n",
20685            )
20686            .unwrap();
20687            run_git(&["add", "spec.rs"]);
20688            run_git(&["commit", "-m", "extend spec"]);
20689            run_git(&["push", "origin", "main"]);
20690            run_git(&["checkout", "fix/review"]);
20691
20692            let mut runner = runner_at(RunStatus::Reviewing);
20693            runner.state.repo = repo.clone();
20694            runner.state.base_commit = base;
20695            runner.state.pr_head = Some("fix/review".into());
20696            runner.state.config.graph.review_rounds = max_rounds;
20697            runner.state.config.graph.worktree_root = Some(tmp.path().join("worktrees"));
20698            runner.state.config.graph.sessions = false;
20699            runner.state.config.disk.min_free_bytes = 0;
20700            runner.state.config.verify.e2e =
20701                vec!["rustc --emit=metadata -o ../check.rmeta check.rs || { echo 'error: could not compile fixture' >&2; exit 101; }".into()];
20702            runner.state.candidates = vec![Candidate {
20703                index: 0,
20704                label: 'A',
20705                agent: "fixture".into(),
20706                branch: "fix/review".into(),
20707                worktree: repo.clone(),
20708                summary: String::new(),
20709                stat: String::new(),
20710                files: 1,
20711                commits: 1,
20712                empty: false,
20713                failed: None,
20714                verified_noop: None,
20715                duration_ms: 0,
20716                folded: false,
20717            }];
20718            runner.state.tally = Some(Tally {
20719                first_choice: BTreeMap::new(),
20720                borda: BTreeMap::new(),
20721                winner: 'A',
20722                rankings: 0,
20723                unanimous_initial: true,
20724                deliberated: false,
20725                changed_votes: 0,
20726                unanimous_final: true,
20727                tie_break: None,
20728                judges: 0,
20729                present: 0,
20730                quorum: 0,
20731                met_quorum: true,
20732                uncontested: Some("review-only".into()),
20733            });
20734            let mut last = review_round(false, 0, 1, 1, false, false);
20735            last.head = reviewed.clone();
20736            last.e2e_deferred = true;
20737            last.e2e[0].resource_blocked = true;
20738            last.e2e[0].output_tail = "waiting for the shared build cache".into();
20739            if max_rounds == 1 {
20740                last.e2e.clear();
20741            }
20742            runner.state.reviews = vec![last];
20743            let calls = tmp.path().join("fix-calls");
20744            let mut fixer = spec("repair");
20745            fixer
20746                .env
20747                .insert("FIX_CALLS".into(), calls.display().to_string());
20748            fixer.command = vec!["sh".into(), "-c".into(), r#"
20749                echo fix >> "$FIX_CALLS"
20750                grep -q E0063 "$MAGI_PROMPT_FILE" || exit 1
20751                grep -q max_turn_secs "$MAGI_PROMPT_FILE" || exit 1
20752                printf 'include!("spec.rs"); fn main() { let _ = AgentSpec { max_turn_secs: 0 }; }\n' > check.rs
20753                printf '{"addressed":[],"rejected":[],"notes":"initialize the new field"}\n'
20754            "#.into()];
20755            let mut reviewer = spec("reviewer");
20756            reviewer.command = vec![
20757                "sh".into(),
20758                "-c".into(),
20759                "printf '%s\\n' '{\"summary\":\"clean\",\"vote\":\"approve\",\"findings\":[]}'"
20760                    .into(),
20761            ];
20762            runner.roles.fixer = Some(vec![fixer]);
20763            runner.roles.reviewers = vec![reviewer];
20764
20765            runner.sync_to_base().await.unwrap();
20766            let rebased = git::rev_parse(&repo, "HEAD").await.unwrap();
20767            assert_ne!(rebased, reviewed);
20768            assert!(runner.state.base_sync.as_ref().unwrap().conflict.is_none());
20769            if reentry {
20770                // Simulate a crash after catch-up evidence was saved but before
20771                // dispatching the fixer. No dedicated pending-repair field exists.
20772                let shell = runner.state.config.shell();
20773                runner
20774                    .stop_unresolved_review("deferred verification", &shell, &repo)
20775                    .await
20776                    .unwrap();
20777                assert!(runner.catchup_repairable());
20778                assert!(!calls.exists());
20779                runner.state = RunState::load(&runner.state.id).unwrap();
20780                assert!(runner.catchup_repairable());
20781            }
20782            runner.review_loop().await.unwrap();
20783            let first = &runner.state.reviews[0];
20784            assert_eq!(first.head, reviewed);
20785            assert_eq!(first.verified_head.as_deref(), Some(rebased.as_str()));
20786            assert!(
20787                first.verify_retried,
20788                "a reproduced build failure gets exactly one retry: {:?}",
20789                first.e2e
20790            );
20791            assert!(first.e2e[0].output_tail.contains("E0063"));
20792            assert!(!runner.catchup_repairable());
20793            if max_rounds == 2 {
20794                assert_eq!(
20795                    runner.state.status,
20796                    RunStatus::Gating,
20797                    "{}",
20798                    crate::report::run(&runner.state)
20799                );
20800                assert_eq!(runner.state.reviews.len(), 2);
20801                assert!(first.fix.as_ref().unwrap().committed);
20802                assert!(runner.state.reviews[1].clean);
20803                assert_eq!(
20804                    runner
20805                        .state
20806                        .reviews
20807                        .iter()
20808                        .filter(|r| r.fix.is_some())
20809                        .count(),
20810                    1
20811                );
20812                assert_eq!(std::fs::read_to_string(&calls).unwrap().lines().count(), 1);
20813                assert_eq!(runner.state.reviews[1].e2e_status(), E2eStatus::Passed);
20814            } else {
20815                assert_eq!(runner.state.status, RunStatus::Blocked);
20816                assert!(first.fix.is_none());
20817                assert!(!calls.exists());
20818                let recovery = crate::recovery::assess(
20819                    "prior",
20820                    Some(&runner.state),
20821                    &runner.landing_base(),
20822                    crate::recovery::Relation::Diverged,
20823                );
20824                assert_eq!(recovery.mode, crate::recovery::Mode::FullReview);
20825                let reasons = recovery.reasons.join("\n");
20826                assert!(
20827                    reasons.contains(&reviewed) && reasons.contains(&rebased),
20828                    "{reasons}"
20829                );
20830                assert!(
20831                    reasons.contains("base sync to")
20832                        && reasons.contains("additional review is required"),
20833                    "{reasons}"
20834                );
20835            }
20836        }
20837    }
20838
20839    /// A resumed run — a fresh `Runner`, `self.state.reviews` already
20840    /// holding the round `stop_reviewing` left `ResourceBlocked` from a
20841    /// prior process — must not sit at `Reviewing` forever: `review_loop`'s
20842    /// own top-of-function fast path (`review_conclusion`) correctly reads
20843    /// this shape as `None` rather than guessing `Blocked`, and the loop's
20844    /// own `for` range is empty once the round budget is spent, so
20845    /// `review_loop` must retry the check itself rather than silently doing
20846    /// nothing. Reaches the exact same retry `stop_reviewing_retries_a_*`
20847    /// above exercises directly, but through `review_loop`'s own entry point
20848    /// this time, proving the wiring between the two rather than just the
20849    /// retry logic in isolation.
20850    #[tokio::test]
20851    async fn a_resumed_review_loop_retries_a_last_round_left_resource_blocked() {
20852        crate::run::pin_test_home();
20853        let home = crate::run::home();
20854
20855        let tmp = tempfile::tempdir().expect("tempdir");
20856        let repo = tmp.path().join("repo");
20857        std::fs::create_dir_all(&repo).unwrap();
20858        init_repo(&repo);
20859        let head = crate::git::rev_parse(&repo, "HEAD")
20860            .await
20861            .expect("rev-parse");
20862        let cache_dir = tmp.path().join("target");
20863
20864        let mut config = Config::default();
20865        config.verify.e2e = vec![format!(
20866            "CARGO_TARGET_DIR='{}' test -f README.md",
20867            cache_dir.display()
20868        )];
20869        config.graph.review_rounds = 1;
20870        config.graph.timeout_verify = Some(2);
20871        config.graph.timeout_resource_wait = 2;
20872
20873        let other = crate::cache::Owner::here("other-run", "e2e", "e2e", &repo, "deadbeef");
20874        let held = match crate::cache::try_acquire(&home, &cache_dir, &other)
20875            .expect("no io error acquiring directly")
20876        {
20877            crate::cache::AcquireOutcome::Acquired(g) => g,
20878            crate::cache::AcquireOutcome::Busy(b) => {
20879                panic!("expected the direct acquire to win the lease first: {b:?}")
20880            }
20881        };
20882
20883        let mut state = RunState::new(
20884            repo.clone(),
20885            "main".to_owned(),
20886            head.clone(),
20887            "task".to_owned(),
20888            config,
20889        );
20890        state.candidates = vec![Candidate {
20891            index: 0,
20892            label: 'A',
20893            agent: "alpha".to_owned(),
20894            branch: "does-not-exist".to_owned(),
20895            worktree: repo.clone(),
20896            summary: String::new(),
20897            stat: String::new(),
20898            files: 0,
20899            commits: 0,
20900            empty: false,
20901            failed: None,
20902            verified_noop: None,
20903            duration_ms: 0,
20904            folded: false,
20905        }];
20906        state.tally = Some(Tally {
20907            first_choice: BTreeMap::from([('A', 1)]),
20908            borda: BTreeMap::new(),
20909            winner: 'A',
20910            rankings: 1,
20911            unanimous_initial: true,
20912            deliberated: false,
20913            changed_votes: 0,
20914            unanimous_final: true,
20915            tie_break: None,
20916            judges: 0,
20917            present: 0,
20918            quorum: 0,
20919            met_quorum: true,
20920            uncontested: Some("only candidate A produced a change".to_owned()),
20921        });
20922        // The exact shape a prior process's `stop_reviewing` would have left
20923        // on disk: the round budget's last round, a real attempt already
20924        // made and already resource-blocked.
20925        state.reviews = vec![ReviewRound {
20926            round: 1,
20927            head: head.clone(),
20928            verified_head: Some(head.clone()),
20929            verified_at: Some(jiff::Timestamp::now()),
20930            reviews: Vec::new(),
20931            e2e: vec![CommandOutcome {
20932                command: format!(
20933                    "CARGO_TARGET_DIR='{}' test -f README.md",
20934                    cache_dir.display()
20935                ),
20936                code: None,
20937                output_tail: "waiting for the shared build cache".to_owned(),
20938                duration_ms: 0,
20939                resource_blocked: true,
20940            }],
20941            fix: None,
20942            blocking: 1,
20943            answered: 1,
20944            expected: 1,
20945            clean: false,
20946            verify_retried: false,
20947            e2e_deferred: false,
20948            e2e_defer_reason: None,
20949            progressed: false,
20950            vote_split: false,
20951            reconsideration: Vec::new(),
20952            verdict: None,
20953        }];
20954
20955        let first_attempt_at = state.reviews[0].verified_at.expect("set above");
20956        let mut runner = Runner {
20957            state,
20958            roles: ResolvedRoles {
20959                implementers: Vec::new(),
20960                judges: Vec::new(),
20961                reviewers: Vec::new(),
20962                fixer: None,
20963                conductor: conductor(),
20964                implementer_roster: Vec::new(),
20965                judge_roster: Vec::new(),
20966                reviewer_roster: Vec::new(),
20967            },
20968            sem: Arc::new(Semaphore::new(1)),
20969            pause: Pause::new(),
20970            interrupt: Pause::new(),
20971        };
20972
20973        // The lease is still held throughout, so this reentry's own retry is
20974        // also contended — proving `review_loop` actually tried again (not
20975        // that it happened to succeed) is what the timestamp comparison
20976        // below is for.
20977        runner.review_loop().await.expect("review_loop");
20978
20979        assert_eq!(
20980            runner.state.reviews.len(),
20981            1,
20982            "no new round was started on top of the unresolved one: {:?}",
20983            runner.state.reviews
20984        );
20985        let last = &runner.state.reviews[0];
20986        assert_eq!(
20987            last.e2e_status(),
20988            E2eStatus::ResourceBlocked,
20989            "still contended: {last:?}"
20990        );
20991        assert!(
20992            last.verified_at.expect("still known") > first_attempt_at,
20993            "review_loop must have actually retried the check, not left it exactly as found"
20994        );
20995        assert_ne!(
20996            runner.state.status,
20997            RunStatus::Blocked,
20998            "a resumed run must not read leftover contention as a verdict on the patch: {:?}",
20999            runner.state.status
21000        );
21001
21002        held.release();
21003    }
21004
21005    #[tokio::test]
21006    async fn stopped_landing_resumes_from_structured_pr_even_with_legacy_gating_status() {
21007        crate::run::pin_test_home();
21008        for status in [RunStatus::Blocked, RunStatus::Gating] {
21009            let mut runner = runner_at(status);
21010            runner.state.config.graph.land = true;
21011            runner.state.gate_ran = true;
21012            runner.state.merge = Some(MergeOutcome {
21013                mode: MergeMode::Pr,
21014                ok: false,
21015                detail: "the fixer produced no commit".into(),
21016                empty: false,
21017            });
21018            let url = "https://example.invalid/o/r/pull/1";
21019            runner.state.pr = Some(crate::run::PrRecord {
21020                url: url.into(),
21021                number: 1,
21022                state: "open".into(),
21023                checks: "red".into(),
21024                round: 1,
21025                rounds: 1,
21026                red_at_merge: Vec::new(),
21027            });
21028            runner.state.save().unwrap();
21029            // The nonexistent repository makes observation fail before a forge
21030            // command can run. We still must reach land, rather than silently
21031            // return from the graph with its old gate and merge records.
21032            runner.execute_graph().await.unwrap();
21033            assert_eq!(runner.state.status, RunStatus::Blocked);
21034            assert!(runner.state.land_sync_pending);
21035            assert!(runner.state.events.iter().any(|event|
21036                event.node == "land" && event.message == format!("watching {url}")));
21037            assert_eq!(
21038                runner.state.merge.as_ref().unwrap().detail,
21039                "the fixer produced no commit"
21040            );
21041        }
21042    }
21043
21044    #[tokio::test]
21045    async fn a_run_resumed_mid_landing_reenters_land_instead_of_opening_a_second_pull_request() {
21046        crate::run::pin_test_home();
21047        let tmp = tempfile::tempdir().expect("tempdir");
21048        let repo = tmp.path().join("repo");
21049        std::fs::create_dir_all(&repo).unwrap();
21050        init_repo(&repo);
21051
21052        let mut config = Config::default();
21053        config.merge.mode = MergeMode::Pr;
21054        config.graph.land = true;
21055        config.graph.land_approval = false;
21056
21057        let mut state = RunState::new(
21058            repo.clone(),
21059            "main".to_owned(),
21060            "deadbeef".to_owned(),
21061            "task".to_owned(),
21062            config,
21063        );
21064        state.candidates = vec![Candidate {
21065            index: 0,
21066            label: 'A',
21067            agent: "alpha".to_owned(),
21068            branch: "does-not-exist".to_owned(),
21069            worktree: repo.clone(),
21070            summary: String::new(),
21071            stat: String::new(),
21072            files: 0,
21073            commits: 0,
21074            empty: false,
21075            failed: None,
21076            verified_noop: None,
21077            duration_ms: 0,
21078            folded: false,
21079        }];
21080        state.tally = Some(Tally {
21081            first_choice: BTreeMap::from([('A', 1)]),
21082            borda: BTreeMap::new(),
21083            winner: 'A',
21084            rankings: 1,
21085            unanimous_initial: true,
21086            deliberated: false,
21087            changed_votes: 0,
21088            unanimous_final: true,
21089            tie_break: None,
21090            judges: 0,
21091            present: 0,
21092            quorum: 0,
21093            met_quorum: true,
21094            uncontested: Some("only candidate A produced a change".to_owned()),
21095        });
21096        state.reviews = vec![ReviewRound {
21097            round: 1,
21098            head: "deadbeef".to_owned(),
21099            verified_head: None,
21100            verified_at: None,
21101            reviews: Vec::new(),
21102            e2e: Vec::new(),
21103            fix: None,
21104            blocking: 0,
21105            answered: 0,
21106            expected: 0,
21107            clean: true,
21108            verify_retried: false,
21109            e2e_deferred: false,
21110            e2e_defer_reason: None,
21111            progressed: false,
21112            vote_split: false,
21113            reconsideration: Vec::new(),
21114            verdict: None,
21115        }];
21116        state.gate = vec![CommandOutcome {
21117            command: "test".to_owned(),
21118            code: Some(0),
21119            output_tail: String::new(),
21120            duration_ms: 0,
21121            resource_blocked: false,
21122        }];
21123        state.gate_ran = true;
21124        // A first pass through `merge` already pushed and opened this pull
21125        // request; `status` is `Landing` because a previous call into `land`
21126        // parked or was interrupted before it reached a terminal outcome.
21127        state.status = RunStatus::Landing;
21128        state.merge = Some(MergeOutcome {
21129            mode: MergeMode::Pr,
21130            ok: true,
21131            detail: "https://example.invalid/x/y/pull/1".to_owned(),
21132            empty: false,
21133        });
21134
21135        // The Landing-resume shortcut calls `run_land` directly rather than
21136        // through `merge`, which is exactly the call site that used to skip
21137        // `settle_questions` - see the fixture below.
21138        ask_test_home();
21139        // A real resume loads a persisted run; driver ownership is checked
21140        // against that record before the Landing shortcut is entered.
21141        state.save().expect("save landing run");
21142        let store = ask::Questions::open();
21143        let q = ask_open_question(&store, &state.id);
21144
21145        let mut runner = Runner {
21146            state,
21147            roles: ResolvedRoles {
21148                implementers: Vec::new(),
21149                judges: Vec::new(),
21150                reviewers: Vec::new(),
21151                fixer: None,
21152                conductor: conductor(),
21153                implementer_roster: Vec::new(),
21154                judge_roster: Vec::new(),
21155                reviewer_roster: Vec::new(),
21156            },
21157            sem: Arc::new(Semaphore::new(1)),
21158            pause: Pause::new(),
21159            interrupt: Pause::new(),
21160        };
21161
21162        // `execute`, not `merge` directly: the Landing-resume shortcut lives
21163        // at the top of `execute`, not inside `merge` (see `execute`'s doc)
21164        // exactly because `review_loop` would otherwise clobber the marker
21165        // first.
21166        runner.execute().await.expect("execute");
21167
21168        assert_eq!(
21169            runner.state.merge.as_ref().map(|m| m.detail.as_str()),
21170            Some("https://example.invalid/x/y/pull/1"),
21171            "reentry must not push again or open a second pull request over the \
21172             one `land` is already watching"
21173        );
21174        assert_ne!(
21175            runner.state.status,
21176            RunStatus::Landing,
21177            "land could not actually reach the fake pull request, so it must \
21178             have given up rather than left the run silently parked forever"
21179        );
21180        // `land` could not reach the fake pull request, so it gave up into
21181        // `Blocked` - still resumable, so the question must not have been
21182        // swept just because this branch now also calls `settle_questions`.
21183        assert_eq!(runner.state.status, RunStatus::Blocked);
21184        assert!(
21185            store.get(&q.id).unwrap().status.open(),
21186            "Blocked is still alive; settle_questions must have been a no-op here"
21187        );
21188    }
21189
21190    fn state_with_round(round: ReviewRound) -> RunState {
21191        let mut s = RunState::new(
21192            PathBuf::from("/repo"),
21193            "main".to_owned(),
21194            "abc1234".to_owned(),
21195            "add retries".to_owned(),
21196            Config::default(),
21197        );
21198        s.reviews = vec![round];
21199        s
21200    }
21201
21202    fn finding(id: &str, severity: Severity, title: &str) -> crate::verdict::Finding {
21203        crate::verdict::Finding {
21204            id: id.to_owned(),
21205            severity,
21206            file: None,
21207            line: None,
21208            title: title.to_owned(),
21209            detail: String::new(),
21210        }
21211    }
21212
21213    #[test]
21214    fn pr_body_names_open_findings_and_declined_ones() {
21215        let round = ReviewRound {
21216            round: 2,
21217            head: "deadbee".to_owned(),
21218            verified_head: None,
21219            verified_at: None,
21220            reviews: vec![ReviewRecord {
21221                attempts: 0,
21222                reviewer: 1,
21223                agent: "alpha".to_owned(),
21224                summary: String::new(),
21225                findings: vec![finding("R2-1-1", Severity::Minor, "unused import")],
21226                vote: None,
21227                failed: None,
21228                duration_ms: 0,
21229            }],
21230            e2e: vec![CommandOutcome {
21231                command: "cargo test".to_owned(),
21232                code: Some(0),
21233                output_tail: String::new(),
21234                duration_ms: 0,
21235                resource_blocked: false,
21236            }],
21237            verify_retried: false,
21238            e2e_deferred: false,
21239            e2e_defer_reason: None,
21240            fix: Some(FixRecord {
21241                agent: "alpha".to_owned(),
21242                addressed: Vec::new(),
21243                rejected: vec![crate::verdict::Rejection {
21244                    id: "R1-1-1".to_owned(),
21245                    why: "not reachable from any caller".to_owned(),
21246                }],
21247                notes: String::new(),
21248                committed: true,
21249                failed: None,
21250                duration_ms: 0,
21251                continuation: None,
21252            }),
21253            blocking: 0,
21254            answered: 1,
21255            expected: 1,
21256            clean: false,
21257            progressed: true,
21258            vote_split: false,
21259            reconsideration: Vec::new(),
21260            verdict: None,
21261        };
21262        let state = state_with_round(round);
21263        let body = pr_message(&state, 'A').body;
21264
21265        assert!(body.contains("add retries"), "the task must still be there");
21266        assert!(body.contains("R2-1-1"), "{body}");
21267        assert!(!body.contains("unused import"), "{body}");
21268        assert!(body.contains("R1-1-1"), "the declined finding: {body}");
21269        assert!(
21270            !body.contains("not reachable from any caller"),
21271            "the reason it was declined: {body}"
21272        );
21273    }
21274
21275    #[test]
21276    fn pr_body_says_nothing_extra_when_the_round_was_clean() {
21277        let round = ReviewRound {
21278            round: 1,
21279            head: "deadbee".to_owned(),
21280            verified_head: None,
21281            verified_at: None,
21282            reviews: vec![ReviewRecord {
21283                attempts: 0,
21284                reviewer: 1,
21285                agent: "alpha".to_owned(),
21286                summary: String::new(),
21287                findings: Vec::new(),
21288                vote: None,
21289                failed: None,
21290                duration_ms: 0,
21291            }],
21292            e2e: Vec::new(),
21293            verify_retried: false,
21294            e2e_deferred: false,
21295            e2e_defer_reason: None,
21296            fix: None,
21297            blocking: 0,
21298            answered: 1,
21299            expected: 1,
21300            clean: true,
21301            progressed: false,
21302            vote_split: false,
21303            reconsideration: Vec::new(),
21304            verdict: None,
21305        };
21306        let state = state_with_round(round);
21307        let body = pr_message(&state, 'A').body;
21308        assert!(!body.contains("Open review findings"), "{body}");
21309        assert!(!body.contains("Declined"), "{body}");
21310    }
21311
21312    fn state_with_summary(instruction: &str, summary: &str) -> RunState {
21313        let mut state = RunState::new(
21314            PathBuf::from("/repo"),
21315            "main".to_owned(),
21316            "abc1234".to_owned(),
21317            instruction.to_owned(),
21318            Config::default(),
21319        );
21320        // These fixtures test execution flow, independent of host disk capacity.
21321        state.config.disk.min_free_bytes = 0;
21322        state.candidates.push(Candidate {
21323            index: 0,
21324            label: 'A',
21325            agent: "alpha".to_owned(),
21326            branch: "magi/x/A".to_owned(),
21327            worktree: PathBuf::from("/wt"),
21328            summary: summary.to_owned(),
21329            stat: String::new(),
21330            files: 1,
21331            commits: 1,
21332            empty: false,
21333            failed: None,
21334            verified_noop: None,
21335            folded: false,
21336            duration_ms: 0,
21337        });
21338        state
21339    }
21340
21341    fn review_state(subjects: &[&str]) -> RunState {
21342        let mut state = state_with_summary(
21343            "Review the work already on branch `magi/x/A`. There is no task statement: what the change claims to do is whatever its commits say.\n\nfirst\nsecond",
21344            "",
21345        );
21346        state.candidates[0].agent = EXISTING_BRANCH.to_owned();
21347        state.reviewed_commits = Some(subjects.iter().map(|s| (*s).to_owned()).collect());
21348        state
21349    }
21350
21351    /// A branch rebased onto a main that moved past the recorded
21352    /// `base_commit` is titled from its own first commit, never main's.
21353    #[tokio::test]
21354    async fn a_rebased_review_branch_is_titled_from_its_own_commits() {
21355        ask_test_home();
21356        let tmp = tempfile::tempdir().unwrap();
21357        let repo = tmp.path().join("repo");
21358        std::fs::create_dir_all(&repo).unwrap();
21359        init_repo(&repo);
21360        let origin = tmp.path().join("origin.git");
21361        let g = |dir: &Path, args: &[&str]| {
21362            let out = std::process::Command::new("git")
21363                .args(args)
21364                .current_dir(dir)
21365                .quiet()
21366                .output()
21367                .expect("spawn git");
21368            assert!(
21369                out.status.success(),
21370                "git {args:?}: {}",
21371                String::from_utf8_lossy(&out.stderr)
21372            );
21373        };
21374        g(
21375            tmp.path(),
21376            &[
21377                "clone",
21378                "--bare",
21379                "-q",
21380                repo.to_str().unwrap(),
21381                origin.to_str().unwrap(),
21382            ],
21383        );
21384        g(
21385            &repo,
21386            &["remote", "add", "origin", origin.to_str().unwrap()],
21387        );
21388        let c1 = git::rev_parse(&repo, "main").await.unwrap();
21389
21390        // Main moves on; the branch is built on top of the new main.
21391        std::fs::write(repo.join("dep.txt"), "bump\n").unwrap();
21392        g(&repo, &["add", "-A"]);
21393        g(
21394            &repo,
21395            &["commit", "-q", "-m", "chore(deps): update a crate"],
21396        );
21397        g(&repo, &["push", "-q", "origin", "main"]);
21398        g(&repo, &["checkout", "-q", "-b", "feat/own"]);
21399        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21400        g(&repo, &["add", "-A"]);
21401        g(
21402            &repo,
21403            &["commit", "-q", "-m", "fix(daemon): apply a chosen action"],
21404        );
21405        g(&repo, &["checkout", "-q", "main"]);
21406
21407        let start = review_base(&repo, "origin", "main", &c1, "feat/own").await;
21408        assert_eq!(start, git::rev_parse(&repo, "main").await.unwrap());
21409        // Without a readable tracking ref the recorded base's merge base is used.
21410        let fallback = review_base(&repo, "nowhere", "main", &c1, "feat/own").await;
21411        assert_eq!(fallback, c1);
21412
21413        let mut state = review_state(&[
21414            "chore(deps): update a crate",
21415            "fix(daemon): apply a chosen action",
21416        ]);
21417        state.repo = repo.clone();
21418        state.base_branch = "main".to_owned();
21419        state.base_commit = c1;
21420        refresh_reviewed_commits(&mut state, "feat/own").await;
21421        assert_eq!(
21422            state.reviewed_commits,
21423            Some(vec!["fix(daemon): apply a chosen action".to_owned()])
21424        );
21425        assert_eq!(
21426            review_title(&state).as_deref(),
21427            Some("fix(daemon): apply a chosen action")
21428        );
21429        assert_eq!(
21430            leaked_subjects(&state, "feat/own").await,
21431            Some(vec!["chore(deps): update a crate".to_owned()])
21432        );
21433        // A stale tracking ref is still used when the fetch fails, but the
21434        // leak list is withheld.
21435        state.config.merge.remote = "nowhere".to_owned();
21436        assert_eq!(leaked_subjects(&state, "feat/own").await, None);
21437    }
21438
21439    /// A review-only run whose winner is `feat/own`, checked out in `repo`,
21440    /// with `origin/main` one commit behind it, and a summarizing agent that
21441    /// prints whatever `reply` holds.
21442    struct SummaryFixture {
21443        _tmp: tempfile::TempDir,
21444        repo: PathBuf,
21445        reply: PathBuf,
21446        runner: Runner,
21447    }
21448
21449    fn summary_fixture() -> SummaryFixture {
21450        ask_test_home();
21451        let tmp = tempfile::tempdir().unwrap();
21452        let repo = tmp.path().join("repo");
21453        std::fs::create_dir_all(&repo).unwrap();
21454        init_repo(&repo);
21455        let origin = tmp.path().join("origin.git");
21456        let g = |args: &[&str]| {
21457            let out = std::process::Command::new("git")
21458                .args(args)
21459                .current_dir(&repo)
21460                .quiet()
21461                .output()
21462                .expect("spawn git");
21463            assert!(
21464                out.status.success(),
21465                "git {args:?}: {}",
21466                String::from_utf8_lossy(&out.stderr)
21467            );
21468        };
21469        g(&["init", "-q", "--bare", origin.to_str().unwrap()]);
21470        g(&["remote", "add", "origin", origin.to_str().unwrap()]);
21471        g(&["push", "-q", "origin", "main"]);
21472        g(&["checkout", "-q", "-b", "feat/own"]);
21473        std::fs::write(repo.join("own.txt"), "own\n").unwrap();
21474        g(&["add", "-A"]);
21475        g(&["commit", "-q", "-m", "feat(own): add the own file"]);
21476        let reply = tmp.path().join("reply.txt");
21477        std::fs::write(
21478            &reply,
21479            "TITLE: feat(own): add the own file\nAdds `own.txt` so the first summary is read.",
21480        )
21481        .unwrap();
21482        let mut runner = runner_at(RunStatus::Gating);
21483        let mut state = review_state(&["feat(own): add the own file"]);
21484        state.repo = repo.clone();
21485        state.base_branch = "main".to_owned();
21486        state.candidates[0].branch = "feat/own".to_owned();
21487        state.candidates[0].worktree = repo.clone();
21488        let mut author = spec("alpha");
21489        author.command = vec!["sh".into(), "-c".into(), "cat \"$REPLY_FILE\"".into()];
21490        author
21491            .env
21492            .insert("REPLY_FILE".into(), reply.to_string_lossy().into_owned());
21493        state.config.agents = vec![author];
21494        runner.state = state;
21495        SummaryFixture {
21496            _tmp: tmp,
21497            repo,
21498            reply,
21499            runner,
21500        }
21501    }
21502
21503    async fn posted(f: &mut SummaryFixture) -> PrMessage {
21504        let winner = f.runner.state.candidates[0].clone();
21505        f.runner
21506            .guarded_pr_message(&winner, None, true)
21507            .await
21508            .unwrap()
21509            .expect("an English description never parks")
21510    }
21511
21512    fn clean_round_with_minor_findings() -> ReviewRound {
21513        let mut round = passing_round(2, None);
21514        round.verdict = Some(crate::verdict::ReviewVote::ApproveWithFindings);
21515        round.reviews = vec![ReviewRecord {
21516            attempts: 0,
21517            reviewer: 1,
21518            agent: "alpha".into(),
21519            summary: "内部レビュー".into(),
21520            findings: vec![
21521                finding("R2-1-1", Severity::Minor, "未修正の懸念その一"),
21522                finding("R2-1-2", Severity::Minor, "未修正の懸念その二"),
21523            ],
21524            vote: Some(crate::verdict::ReviewVote::ApproveWithFindings),
21525            failed: None,
21526            duration_ms: 0,
21527        }];
21528        round
21529    }
21530
21531    #[tokio::test]
21532    async fn public_descriptions_cover_minor_findings_from_clean_rounds_in_both_modes() {
21533        for review_only in [false, true] {
21534            let mut f = summary_fixture();
21535            if !review_only {
21536                f.runner.state.candidates[0].agent = "alpha".into();
21537            }
21538            f.runner.state.reviews = vec![clean_round_with_minor_findings()];
21539            assert!(f.runner.state.reviews[0].clean);
21540            assert!(f.runner.state.reviews[0].fix.is_none());
21541            assert!(f.runner.state.open_findings().is_empty());
21542            assert_eq!(
21543                crate::pr_outcome::required_finding_ids(&f.runner.state),
21544                ["R2-1-1", "R2-1-2"]
21545            );
21546            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21547            let evidence = crate::pr_outcome::summary_evidence(
21548                &f.runner.state,
21549                crate::pr_outcome::FinalHead::At(&head),
21550            );
21551            assert!(
21552                evidence.contains("未修正の懸念その一") && evidence.contains("未修正の懸念その二")
21553            );
21554            // The ordinary reply omits both IDs: no cached description is adopted.
21555            let omitted = posted(&mut f).await;
21556            assert!(omitted.body.contains("No description"));
21557            assert!(omitted.body.contains("`R2-1-1` [Minor]"));
21558            assert!(omitted.body.contains("`R2-1-2` [Minor]"));
21559            assert!(f.runner.state.change_summary.is_none());
21560            std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nAdds the own file. R2-1-1 [Minor]: improve documentation for callers. R2-1-2 [Minor]: consider the remaining edge case.").unwrap();
21561            let described = posted(&mut f).await;
21562            assert!(described.body.contains("improve documentation for callers"));
21563            assert!(described.body.contains("remaining edge case"));
21564            assert!(described.body.contains("## Open review findings"));
21565            assert!(!described.body.contains("未修正"));
21566        }
21567    }
21568
21569    async fn rewrite_fixture_with_findings(review_only: bool, reply: &str) -> SummaryFixture {
21570        let mut f = summary_fixture();
21571        if !review_only {
21572            f.runner.state.candidates[0].agent = "alpha".into();
21573        }
21574        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21575        let mut fixed = round_with_notes(1, Some("日本語の内部修正報告"));
21576        fixed
21577            .fix
21578            .as_mut()
21579            .unwrap()
21580            .rejected
21581            .push(crate::verdict::Rejection {
21582                id: "R1-1-1".into(),
21583                why: "呼び出されないコードのため辞退".into(),
21584            });
21585        let mut clean = clean_round_with_minor_findings();
21586        clean.verified_head = Some(head.clone());
21587        f.runner.state.reviews = vec![fixed, clean];
21588        ok_gate(&mut f.runner.state, Some(&head));
21589        std::fs::write(&f.reply, "TITLE: fix: public description\n日本語の公開用説明です。R1-1-1 は辞退。R2-1-1 と R2-1-2 の懸念が残ります。").unwrap();
21590        f.runner.state.config.agents.push(scripted("beta", reply));
21591        f.runner.state.config.roles.language_judge =
21592            Some(crate::config::AgentChoice::One("beta".into()));
21593        f
21594    }
21595
21596    #[tokio::test]
21597    async fn posting_rewrite_retains_structured_evidence_and_finding_explanations() {
21598        for review_only in [false, true] {
21599            let mut f = rewrite_fixture_with_findings(review_only,
21600                "TITLE: fix: public description\nR1-1-1: declined because no caller reaches this code. R2-1-1 [Minor]: documentation remains incomplete. R2-1-2 [Minor]: an edge case needs follow-up.").await;
21601            let m = posted(&mut f).await;
21602            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21603            let expected =
21604                public_pr_evidence(&f.runner.state, crate::pr_outcome::FinalHead::At(&head));
21605            assert!(m.body.contains(&expected));
21606            assert!(m.body.contains("## Review rounds"));
21607            assert!(
21608                m.body
21609                    .contains("Gate (`verify.gate`): passed on the final commit")
21610            );
21611            assert!(
21612                m.body
21613                    .contains("Tests (`verify.e2e`): passed on the final commit")
21614            );
21615            let c = f.runner.state.change_summary.as_ref().unwrap();
21616            assert!(
21617                m.body
21618                    .contains(&format!("Evidence: final diff `{}`..`{}`", c.base, head))
21619            );
21620            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21621            assert!(crate::pr_outcome::covers_required_findings(
21622                &f.runner.state,
21623                &m.body
21624            ));
21625            assert!(m.body.contains("documentation remains incomplete"));
21626            assert!(!m.body.contains("内部") && !m.body.contains("辞退"));
21627            assert_eq!(f.runner.state.seats["github-text-rewrite"].turns, 1);
21628            assert!(f.runner.state.github_text.is_none());
21629        }
21630    }
21631
21632    #[tokio::test]
21633    async fn posting_rewrite_missing_required_finding_context_is_not_offered_or_published() {
21634        for review_only in [false, true] {
21635            for reply in [
21636                "TITLE: fix: public description\nA short English description.",
21637                "TITLE: fix: public description\nR1-1-1 and R2-1-2 remain; R2-1-1-extra is a different ID.",
21638            ] {
21639                let mut f = rewrite_fixture_with_findings(review_only, reply).await;
21640                let winner = f.runner.state.candidates[0].clone();
21641                let posted = f
21642                    .runner
21643                    .guarded_pr_message(&winner, None, true)
21644                    .await
21645                    .unwrap();
21646                assert!(posted.is_none());
21647                assert!(f.runner.state.parked);
21648                let withheld = f.runner.state.github_text.as_ref().unwrap();
21649                assert!(
21650                    withheld.rewrite.is_none(),
21651                    "an incomplete rewrite must not be selectable"
21652                );
21653                let q = ask::Questions::open()
21654                    .get(withheld.question.as_ref().unwrap())
21655                    .unwrap();
21656                assert!(
21657                    !q.choices
21658                        .contains(&crate::github_text::USE_REWRITE.to_owned())
21659                );
21660                assert!(
21661                    q.detail.contains("R1-1-1")
21662                        && q.detail.contains("R2-1-1")
21663                        && q.detail.contains("R2-1-2")
21664                );
21665            }
21666        }
21667    }
21668
21669    #[tokio::test]
21670    async fn japanese_reports_stay_local_in_both_public_run_modes() {
21671        for review_only in [false, true] {
21672            let mut f = summary_fixture();
21673            if !review_only {
21674                f.runner.state.candidates[0].agent = "alpha".into();
21675            }
21676            f.runner.state.instruction = "内部タスクの説明".into();
21677            f.runner.state.candidates[0].summary = "内部の候補報告".into();
21678            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21679            f.runner.state.reviews = vec![
21680                round_with_notes(1, Some("コミットできません。テストは実行できません。")),
21681                round_with_notes(2, Some("二回目の内部修正報告です。")),
21682                passing_round(3, Some(&head)),
21683            ];
21684            f.runner.state.reviews[0].fix.as_mut().unwrap().addressed = vec!["R1-1-1".into()];
21685            ok_gate(&mut f.runner.state, Some(&head));
21686            let captured = f.repo.join("public-prompt.txt");
21687            let author = &mut f.runner.state.config.agents[0];
21688            author.command = vec![
21689                "sh".into(),
21690                "-c".into(),
21691                "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21692            ];
21693            author.env.insert(
21694                "PROMPT_FILE".into(),
21695                captured.to_string_lossy().into_owned(),
21696            );
21697            let m = posted(&mut f).await;
21698            assert!(m.body.contains("Adds `own.txt`"));
21699            assert!(m.body.contains("Round 1") && m.body.contains("Round 2"));
21700            assert!(m.body.contains("R1-1-1"));
21701            assert!(
21702                m.body
21703                    .contains("Gate (`verify.gate`): passed on the final commit")
21704            );
21705            assert!(
21706                m.body
21707                    .contains("Tests (`verify.e2e`): passed on the final commit")
21708            );
21709            assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21710            assert!(!m.body.contains("内部") && !m.body.contains("できません"));
21711            let prompt = std::fs::read_to_string(captured).unwrap();
21712            assert!(prompt.contains(&format!(
21713                "git diff {} {head}",
21714                f.runner.state.change_summary.as_ref().unwrap().base
21715            )));
21716            assert!(!prompt.contains("二回目") && !prompt.contains("コミットできません"));
21717            assert_eq!(
21718                f.runner.state.reviews[1].fix.as_ref().unwrap().notes,
21719                "二回目の内部修正報告です。"
21720            );
21721        }
21722    }
21723
21724    #[tokio::test]
21725    async fn public_generation_failure_never_falls_back_to_internal_notes_and_resume_retries() {
21726        for review_only in [false, true] {
21727            let mut f = summary_fixture();
21728            if !review_only {
21729                f.runner.state.candidates[0].agent = "alpha".into();
21730            }
21731            f.runner.state.candidates[0].summary = "内部報告を公開しない".into();
21732            f.runner.state.reviews = vec![round_with_notes(1, Some("日本語の修正報告"))];
21733            std::fs::write(&f.reply, "no usable description").unwrap();
21734            let failed = posted(&mut f).await;
21735            assert!(
21736                failed
21737                    .body
21738                    .contains("No description of what this change does could be generated")
21739            );
21740            assert!(failed.body.is_ascii());
21741            assert!(f.runner.state.github_text.is_none());
21742            assert!(f.runner.state.change_summary.is_none());
21743            let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21744            let evidence = crate::pr_outcome::summary_evidence(
21745                &f.runner.state,
21746                crate::pr_outcome::FinalHead::At(&head),
21747            );
21748            let fp = crate::pr_outcome::summary_fingerprint(&f.runner.state, &evidence);
21749            assert!(
21750                crate::run::artifact_path(&f.runner.state, &format!("public-description-{fp}.md"))
21751                    .exists()
21752            );
21753            f.runner.state =
21754                serde_json::from_str(&serde_json::to_string(&f.runner.state).unwrap()).unwrap();
21755            std::fs::write(
21756                &f.reply,
21757                "TITLE: feat(own): add the own file\nAdds the committed own file.",
21758            )
21759            .unwrap();
21760            let retried = posted(&mut f).await;
21761            assert!(retried.body.contains("Adds the committed own file."));
21762            assert_eq!(
21763                f.runner.state.reviews[0].fix.as_ref().unwrap().notes,
21764                "日本語の修正報告"
21765            );
21766        }
21767    }
21768
21769    #[tokio::test]
21770    async fn same_head_verification_and_findings_invalidate_public_prose() {
21771        let mut f = summary_fixture();
21772        posted(&mut f).await;
21773        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21774        let original = f
21775            .runner
21776            .state
21777            .change_summary
21778            .as_ref()
21779            .unwrap()
21780            .evidence
21781            .clone();
21782        ok_gate(&mut f.runner.state, Some(&head));
21783        std::fs::write(
21784            &f.reply,
21785            "TITLE: feat(own): add the own file\nDescribes the now verified final change.",
21786        )
21787        .unwrap();
21788        let verified = posted(&mut f).await;
21789        assert!(verified.body.contains("now verified"));
21790        assert_ne!(
21791            f.runner.state.change_summary.as_ref().unwrap().evidence,
21792            original
21793        );
21794        let mut round = round_with_notes(1, Some("内部報告"));
21795        round.reviews.push(ReviewRecord {
21796            attempts: 0,
21797            reviewer: 1,
21798            agent: "alpha".into(),
21799            summary: "内部レビュー".into(),
21800            findings: vec![finding("R1-1-1", Severity::Minor, "残っている懸念")],
21801            vote: None,
21802            failed: None,
21803            duration_ms: 0,
21804        });
21805        round
21806            .fix
21807            .as_mut()
21808            .unwrap()
21809            .rejected
21810            .push(crate::verdict::Rejection {
21811                id: "R1-1-1".into(),
21812                why: "呼び出し元から到達できない".into(),
21813            });
21814        f.runner.state.reviews = vec![round];
21815        // A description omitting the required ID is unusable, even with a title.
21816        let omitted = posted(&mut f).await;
21817        assert!(omitted.body.contains("No description"));
21818        assert!(omitted.body.contains("R1-1-1"));
21819        assert!(!omitted.body.contains("到達"));
21820        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: declined because the code is unreachable from callers; confirm that assumption when adding callers.").unwrap();
21821        let described = posted(&mut f).await;
21822        assert!(described.body.contains("unreachable from callers"));
21823        assert!(described.body.contains("R1-1-1"));
21824        assert!(!described.body.contains("到達"));
21825        let evidence = f
21826            .runner
21827            .state
21828            .change_summary
21829            .as_ref()
21830            .unwrap()
21831            .evidence
21832            .clone();
21833        f.runner.state.reviews[0].fix.as_mut().unwrap().rejected[0].why = "別の根拠".into();
21834        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: decline rationale updated after review.").unwrap();
21835        assert!(posted(&mut f).await.body.contains("rationale updated"));
21836        assert_ne!(
21837            f.runner.state.change_summary.as_ref().unwrap().evidence,
21838            evidence
21839        );
21840        // Old serialized summaries have no evidence fingerprint and must regenerate.
21841        f.runner
21842            .state
21843            .change_summary
21844            .as_mut()
21845            .unwrap()
21846            .evidence
21847            .clear();
21848        std::fs::write(&f.reply, "TITLE: feat(own): add the own file\nR1-1-1 [Minor]: regenerated under the current specification.").unwrap();
21849        assert!(posted(&mut f).await.body.contains("current specification"));
21850    }
21851
21852    #[tokio::test]
21853    async fn a_review_only_run_with_no_fixer_gets_a_summary_of_the_final_diff() {
21854        let mut f = summary_fixture();
21855        let m = posted(&mut f).await;
21856        assert_eq!(m.title, "feat(own): add the own file");
21857        assert!(
21858            m.body
21859                .contains("## Summary\n\nAdds `own.txt` so the first summary is read."),
21860            "{}",
21861            m.body
21862        );
21863        assert!(!m.body.contains("## Review fixes"), "{}", m.body);
21864        assert!(m.body.contains("Evidence: final diff"), "{}", m.body);
21865        assert!(m.body.contains(&format!("magi:run/{}", f.runner.state.id)));
21866        let head = git::rev_parse(&f.repo, "HEAD").await.unwrap();
21867        let base = git::rev_parse(&f.repo, "origin/main").await.unwrap();
21868        let kept = f.runner.state.change_summary.as_ref().unwrap();
21869        assert_eq!(
21870            (kept.head.as_str(), kept.base.as_str()),
21871            (head.as_str(), base.as_str())
21872        );
21873        assert!(!f.runner.state.seats["change-summary"].key.is_empty());
21874    }
21875
21876    #[tokio::test]
21877    async fn review_only_summary_receives_the_configured_github_language() {
21878        let mut f = summary_fixture();
21879        f.runner.state.config.graph.github_language = "ja".into();
21880        let captured = f.repo.join("summary-prompt.txt");
21881        let author = &mut f.runner.state.config.agents[0];
21882        author.command = vec![
21883            "sh".into(),
21884            "-c".into(),
21885            "cat > \"$PROMPT_FILE\"; cat \"$REPLY_FILE\"".into(),
21886        ];
21887        author.env.insert(
21888            "PROMPT_FILE".into(),
21889            captured.to_string_lossy().into_owned(),
21890        );
21891        std::fs::write(
21892            &f.reply,
21893            "TITLE: feat(own): ファイルを追加\nown.txt を追加します。",
21894        )
21895        .unwrap();
21896        let m = posted(&mut f).await;
21897        let prompt = std::fs::read_to_string(captured).unwrap();
21898        assert!(prompt.contains("Write the TITLE line and the Markdown description in Japanese."));
21899        assert!(!prompt.contains("in English"));
21900        assert_eq!(m.title, "feat(own): ファイルを追加");
21901        assert!(m.body.contains("own.txt を追加します。"));
21902    }
21903
21904    #[tokio::test]
21905    async fn review_fixes_come_after_the_whole_change_summary() {
21906        let mut f = summary_fixture();
21907        f.runner.state.reviews = vec![
21908            round_with_notes(1, Some("handled the PATH case")),
21909            round_with_notes(2, None),
21910        ];
21911        let m = posted(&mut f).await;
21912        let summary = m.body.find("## Summary").expect("summary");
21913        let fixes = m.body.find("## Review rounds").expect("review rounds");
21914        assert!(summary < fixes, "{}", m.body);
21915        assert!(!m.body.contains("handled the PATH case"), "{}", m.body);
21916        assert!(m.body.contains("## Final verification"), "{}", m.body);
21917    }
21918
21919    #[tokio::test]
21920    async fn a_changed_final_diff_invalidates_the_earlier_summary() {
21921        let mut f = summary_fixture();
21922        let first = posted(&mut f).await;
21923        let first_head = f.runner.state.change_summary.clone().unwrap().head;
21924        // The same head reuses the record: a different reply is not read.
21925        std::fs::write(&f.reply, "TITLE: feat(own): other\nNot read yet.").unwrap();
21926        let again = posted(&mut f).await;
21927        assert_eq!((again.title, again.body), (first.title, first.body));
21928
21929        let g = |args: &[&str]| {
21930            let out = std::process::Command::new("git")
21931                .args(args)
21932                .current_dir(&f.repo)
21933                .quiet()
21934                .output()
21935                .expect("spawn git");
21936            assert!(out.status.success(), "git {args:?}");
21937        };
21938        std::fs::write(f.repo.join("more.txt"), "more\n").unwrap();
21939        g(&["add", "-A"]);
21940        g(&["commit", "-q", "-m", "fix(own): handle more"]);
21941        let second = posted(&mut f).await;
21942        let kept = f.runner.state.change_summary.clone().unwrap();
21943        assert_ne!(kept.head, first_head);
21944        assert!(second.body.contains("Not read yet."), "{}", second.body);
21945        assert!(!second.body.contains("so the first summary is read"));
21946        assert_eq!(second.title, "feat(own): other");
21947    }
21948
21949    #[tokio::test]
21950    async fn an_unavailable_summary_is_stated_and_never_invented() {
21951        let mut f = summary_fixture();
21952        std::fs::write(&f.reply, "no title line here").unwrap();
21953        let m = posted(&mut f).await;
21954        assert!(
21955            m.body
21956                .contains("No description of what this change does could be generated"),
21957            "{}",
21958            m.body
21959        );
21960        assert!(m.body.contains("internal reports remain in local records"));
21961        assert!(f.runner.state.change_summary.is_none());
21962        assert!(!m.body.contains("## Commits under review"), "{}", m.body);
21963        // A preview asks nobody and claims nothing.
21964        let winner = f.runner.state.candidates[0].clone();
21965        let preview = f
21966            .runner
21967            .guarded_pr_message(&winner, None, false)
21968            .await
21969            .unwrap()
21970            .unwrap();
21971        assert!(!preview.body.contains("## Summary"), "{}", preview.body);
21972    }
21973
21974    #[test]
21975    fn pr_message_review_single_commit_uses_its_subject() {
21976        let state = review_state(&["feat(nats): per-role user"]);
21977        let m = pr_message(&state, 'A');
21978        assert_eq!(m.title, "feat(nats): per-role user");
21979        assert!(!m.body.contains("Review the work already"), "{}", m.body);
21980        assert!(m.body.contains("## Commits under review"), "{}", m.body);
21981    }
21982
21983    #[test]
21984    fn pr_message_review_multi_commit_takes_the_oldest() {
21985        let state = review_state(&["feat: the change", "fix: typo", "fix: again"]);
21986        let m = pr_message(&state, 'A');
21987        assert_eq!(m.title, "feat: the change");
21988        for s in ["feat: the change", "fix: typo", "fix: again"] {
21989            assert!(m.body.contains(&format!("- {s}\n")), "{}", m.body);
21990        }
21991    }
21992
21993    #[test]
21994    fn pr_message_review_without_a_usable_first_subject_is_neutral() {
21995        for first in ["日本語の件名", "", "magi: candidate A (uncommitted work)"] {
21996            let mut state = review_state(&[first, "fix: later fixup"]);
21997            state.candidates[0].branch = "機能/ログイン".to_owned();
21998            let m = pr_message(&state, 'A');
21999            assert!(
22000                m.title.starts_with("chore: land candidate A of run"),
22001                "{}",
22002                m.title
22003            );
22004        }
22005    }
22006
22007    fn facts(commits: &[(&str, &str)], stat: &str) -> BranchFacts {
22008        BranchFacts {
22009            commits: commits
22010                .iter()
22011                .map(|(s, b)| ((*s).to_owned(), (*b).to_owned()))
22012                .collect(),
22013            stat: stat.to_owned(),
22014        }
22015    }
22016
22017    fn round_with_notes(round: usize, notes: Option<&str>) -> ReviewRound {
22018        let mut r = review_round(true, 0, 1, 1, true, true);
22019        r.round = round;
22020        r.fix = notes.map(|n| FixRecord {
22021            agent: "fixer".to_owned(),
22022            addressed: Vec::new(),
22023            rejected: Vec::new(),
22024            notes: n.to_owned(),
22025            committed: true,
22026            failed: None,
22027            duration_ms: 0,
22028            continuation: None,
22029        });
22030        r
22031    }
22032
22033    #[test]
22034    fn pr_message_review_with_branch_facts_uses_commits_and_stat() {
22035        let state = review_state(&["ignored"]);
22036        let f = facts(
22037            &[
22038                (
22039                    "fix(login): resolve PATH on macOS",
22040                    "Login shells skip rc files.",
22041                ),
22042                ("fix: address review", ""),
22043            ],
22044            " src/a.rs | 2 +-\n 1 file changed, 1 insertion(+), 1 deletion(-)",
22045        );
22046        let m = pr_message_with(&state, 'A', Some(&f));
22047        assert_eq!(m.title, "fix(login): resolve PATH on macOS");
22048        assert!(
22049            m.body
22050                .contains("- fix(login): resolve PATH on macOS\n  Login shells skip rc files.\n"),
22051            "{}",
22052            m.body
22053        );
22054        assert!(m.body.contains("- fix: address review\n"), "{}", m.body);
22055        assert!(m.body.contains("## Diff stat"), "{}", m.body);
22056        assert!(m.body.contains("src/a.rs | 2 +-"), "{}", m.body);
22057        for banned in [
22058            "Review the work already",
22059            "no task statement",
22060            "Original task",
22061        ] {
22062            assert!(!m.body.contains(banned), "{banned}: {}", m.body);
22063        }
22064        assert!(m.body.ends_with("magi:candidate-a\n"), "{}", m.body);
22065    }
22066
22067    #[test]
22068    fn pr_message_review_truncates_a_huge_first_commit_body() {
22069        let state = review_state(&["ignored"]);
22070        let f = facts(
22071            &[("feat: big", &"x".repeat(70_000)), ("fix: later", "")],
22072            "s",
22073        );
22074        let m = pr_message_with(&state, 'A', Some(&f));
22075        assert!(m.body.len() < 30_000, "{}", m.body.len());
22076        assert!(m.body.contains("(truncated)"), "{}", m.body.len());
22077        assert!(
22078            m.body.contains("- ... 1 more commit(s)"),
22079            "{}",
22080            m.body.len()
22081        );
22082        assert!(m.body.ends_with("magi:candidate-a\n"));
22083    }
22084
22085    #[test]
22086    fn pr_message_review_without_facts_falls_back_to_recorded_subjects() {
22087        let m = pr_message_with(&review_state(&["feat: x", "fix: y"]), 'A', None);
22088        assert_eq!(m.title, "feat: x");
22089        assert!(m.body.contains("- fix: y\n"), "{}", m.body);
22090        assert!(!m.body.contains("Diff stat"), "{}", m.body);
22091        assert!(!m.body.contains("no task statement"), "{}", m.body);
22092    }
22093
22094    #[test]
22095    fn pr_message_review_titles_from_the_branch_name_when_subjects_are_unusable() {
22096        let mut state = review_state(&["日本語の件名"]);
22097        state.candidates[0].branch = "fix/macos-login-path".to_owned();
22098        assert_eq!(pr_message(&state, 'A').title, "fix/macos-login-path");
22099        state.candidates[0].branch = "機能/ログイン".to_owned();
22100        assert!(
22101            pr_message(&state, 'A')
22102                .title
22103                .starts_with("chore: land candidate A")
22104        );
22105    }
22106
22107    #[test]
22108    fn pr_message_review_fixes_survive_a_clean_final_round() {
22109        let mut state = review_state(&["feat: x"]);
22110        state.reviews = vec![
22111            round_with_notes(1, Some("handled the PATH case")),
22112            round_with_notes(2, None),
22113        ];
22114        let body = pr_message(&state, 'A').body;
22115        assert!(
22116            body.contains("## Review rounds") && !body.contains("handled the PATH case"),
22117            "{body}"
22118        );
22119        assert!(body.contains("### Round 1"), "{body}");
22120
22121        state.reviews = vec![
22122            round_with_notes(1, Some("first fix")),
22123            round_with_notes(2, Some("")),
22124            round_with_notes(3, Some("second fix")),
22125            round_with_notes(4, None),
22126        ];
22127        let body = pr_message(&state, 'A').body;
22128        assert!(
22129            body.contains("### Round 1") && !body.contains("first fix"),
22130            "{body}"
22131        );
22132        assert!(
22133            body.contains("### Round 3") && !body.contains("second fix"),
22134            "{body}"
22135        );
22136        assert!(body.contains("### Round 2"), "{body}");
22137    }
22138
22139    const FINAL: &str = "abcdef1234567890";
22140
22141    fn passing_round(round: usize, verified: Option<&str>) -> ReviewRound {
22142        let mut r = round_with_notes(round, None);
22143        r.verified_head = verified.map(str::to_owned);
22144        r
22145    }
22146
22147    fn ok_gate(state: &mut RunState, head: Option<&str>) {
22148        state.gate = vec![CommandOutcome {
22149            command: "gate".to_owned(),
22150            code: Some(0),
22151            output_tail: String::new(),
22152            duration_ms: 0,
22153            resource_blocked: false,
22154        }];
22155        state.gate_ran = true;
22156        state.gate_head = head.map(str::to_owned);
22157    }
22158
22159    fn body_at(state: &RunState) -> String {
22160        pr_message_raw(
22161            state,
22162            'A',
22163            None,
22164            crate::pr_outcome::FinalHead::At(FINAL),
22165            ChangeNote::NotAsked,
22166        )
22167        .body
22168    }
22169
22170    const SANDBOXED: &str = "Handled the PATH case.\nI could not commit because of the sandbox. HTTP tests could not run in the sandbox.";
22171
22172    #[test]
22173    fn rescued_commit_and_later_verification_override_the_fixers_limits() {
22174        let mut state = review_state(&["feat: x"]);
22175        state.reviews = vec![
22176            round_with_notes(1, Some(SANDBOXED)),
22177            passing_round(2, Some(FINAL)),
22178        ];
22179        ok_gate(&mut state, Some(FINAL));
22180        let body = body_at(&state);
22181        assert!(!body.contains("Handled the PATH case."), "{body}");
22182        assert!(!body.contains("could not commit"), "{body}");
22183        assert!(!body.contains("could not run"), "{body}");
22184        assert!(
22185            body.contains("The fix is committed on the branch"),
22186            "{body}"
22187        );
22188        assert!(!body.contains("sandbox"), "{body}");
22189        assert!(
22190            body.contains("Tests (`verify.e2e`): passed on the final commit"),
22191            "{body}"
22192        );
22193        assert!(
22194            body.contains("Gate (`verify.gate`): passed on the final commit"),
22195            "{body}"
22196        );
22197        // The raw report is untouched in the record.
22198        assert!(
22199            state.reviews[0]
22200                .fix
22201                .as_ref()
22202                .unwrap()
22203                .notes
22204                .contains("could not commit")
22205        );
22206    }
22207
22208    #[test]
22209    fn unresolved_commit_and_test_limits_stay_and_nothing_claims_success() {
22210        let mut state = review_state(&["feat: x"]);
22211        let mut r1 = round_with_notes(1, Some(SANDBOXED));
22212        r1.fix.as_mut().unwrap().committed = false;
22213        let mut r2 = passing_round(2, Some(FINAL));
22214        r2.e2e[0].code = Some(1);
22215        state.reviews = vec![r1, r2];
22216        let body = body_at(&state);
22217        assert!(!body.contains("could not commit"), "{body}");
22218        assert!(!body.contains("could not run"), "{body}");
22219        assert!(body.contains("This round produced no commit"), "{body}");
22220        assert!(body.contains("Tests (`verify.e2e`): FAILED"), "{body}");
22221        assert!(body.contains("Gate (`verify.gate`): not run"), "{body}");
22222        assert!(!body.contains("passed"), "{body}");
22223    }
22224
22225    #[test]
22226    fn verification_on_an_earlier_commit_does_not_supersede_test_limits() {
22227        let mut state = review_state(&["feat: x"]);
22228        state.reviews = vec![
22229            round_with_notes(1, Some(SANDBOXED)),
22230            passing_round(2, Some("0123456789abcdef")),
22231        ];
22232        ok_gate(&mut state, Some("0123456789abcdef"));
22233        let body = body_at(&state);
22234        assert!(!body.contains("HTTP tests could not run"), "{body}");
22235        // The commit claim is still settled by the structured record.
22236        assert!(!body.contains("could not commit"), "{body}");
22237        assert!(
22238            body.contains("passed on an earlier commit (`0123456`)"),
22239            "{body}"
22240        );
22241        assert!(!body.contains("passed on the final commit"), "{body}");
22242
22243        // A gate recorded before `gate_head` existed is "not recorded".
22244        ok_gate(&mut state, None);
22245        let body = body_at(&state);
22246        assert!(
22247            body.contains("passed on a commit that was not recorded"),
22248            "{body}"
22249        );
22250    }
22251
22252    #[test]
22253    fn pr_message_implementation_run_keeps_its_shape_and_marker() {
22254        let state = state_with_summary(
22255            "add retries to the client",
22256            "TITLE: feat: retries\n\nDid it.",
22257        );
22258        let m = pr_message_with(&state, 'A', Some(&facts(&[("x", "")], "s")));
22259        assert_eq!(m.title, "feat: retries");
22260        assert!(
22261            m.body.contains("<summary>Original task</summary>"),
22262            "{}",
22263            m.body
22264        );
22265        assert!(!m.body.contains("Commits under review"), "{}", m.body);
22266        assert!(
22267            m.body
22268                .ends_with(&format!("magi:run/{} magi:candidate-a\n", state.id)),
22269            "{}",
22270            m.body
22271        );
22272    }
22273
22274    #[test]
22275    fn pr_message_review_bounds_a_long_english_subject() {
22276        let long = format!("feat: {}", "word ".repeat(100));
22277        let m = pr_message(&review_state(&[&long]), 'A');
22278        assert!(m.title.starts_with("feat: word"), "{}", m.title);
22279        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22280    }
22281
22282    #[test]
22283    fn pr_message_implementation_run_is_unchanged_by_review_support() {
22284        let state = state_with_summary("add retries\n\ndetails", "- did some things");
22285        let m = pr_message(&state, 'A');
22286        assert_eq!(m.title, "add retries");
22287        assert!(m.body.contains("<summary>Original task</summary>"));
22288        assert!(!m.body.contains("Commits under review"));
22289        assert_eq!(landing_subject_source(&state), state.instruction);
22290    }
22291
22292    #[test]
22293    fn review_run_squash_subject_is_the_change_not_the_prompt() {
22294        let state = review_state(&["feat: the change", "fix: typo"]);
22295        let source = landing_subject_source(&state);
22296        assert_eq!(land::merge_subject("", &source), "feat: the change");
22297        assert_eq!(
22298            land::merge_subject("magi: candidate A (uncommitted work)", &source),
22299            "feat: the change"
22300        );
22301        // An operator's rename still wins.
22302        assert_eq!(
22303            land::merge_subject("feat: renamed by hand", &source),
22304            "feat: renamed by hand"
22305        );
22306        let blank = review_state(&["日本語"]);
22307        assert!(
22308            land::merge_subject("", &landing_subject_source(&blank)).starts_with("chore: land")
22309        );
22310    }
22311
22312    #[test]
22313    fn review_run_drops_a_prompt_shaped_pr_title_at_landing() {
22314        let state = review_state(&["feat: the change"]);
22315        let source = landing_subject_source(&state);
22316        let old = "Review the work already on branch `magi/x/A`. There is no task statement";
22317        assert_eq!(
22318            land::merge_subject(landing_title(&state, old), &source),
22319            "feat: the change"
22320        );
22321        assert_eq!(landing_title(&state, "feat: renamed"), "feat: renamed");
22322        let task = state_with_summary("add retries", "");
22323        assert_eq!(landing_title(&task, old), old);
22324    }
22325
22326    #[test]
22327    fn pr_message_describes_the_change_not_the_task() {
22328        let state = state_with_summary(
22329            "今回やってほしいこと: results projector を直す",
22330            "TITLE: fix(web): batch the runs list reads\n- reads run.json once\n- risk: none",
22331        );
22332        let m = pr_message(&state, 'A');
22333        assert_eq!(m.title, "fix(web): batch the runs list reads");
22334        assert!(
22335            m.body.starts_with("## Summary\n\n- reads run.json once"),
22336            "{}",
22337            m.body
22338        );
22339        assert!(!m.body.contains("TITLE:"), "{}", m.body);
22340        let task_at = m.body.find("今回やってほしいこと").unwrap();
22341        let details_at = m.body.find("<details>").unwrap();
22342        assert!(
22343            details_at < task_at,
22344            "the task lives inside <details>: {}",
22345            m.body
22346        );
22347        assert!(m.body.contains(&format!("magi:run/{}", state.id)));
22348        assert!(m.body.contains("magi:candidate-a"));
22349    }
22350
22351    #[test]
22352    fn pr_message_falls_back_to_the_task_without_a_title_line() {
22353        let state = state_with_summary("\n\nadd retries\n\ndetails", "- did some things");
22354        let m = pr_message(&state, 'A');
22355        assert_eq!(m.title, "add retries");
22356        assert!(
22357            m.body.contains("## Summary\n\n- did some things"),
22358            "{}",
22359            m.body
22360        );
22361
22362        let none = RunState::new(
22363            PathBuf::from("/repo"),
22364            "main".to_owned(),
22365            "abc1234".to_owned(),
22366            "add retries".to_owned(),
22367            Config::default(),
22368        );
22369        let m = pr_message(&none, 'A');
22370        assert_eq!(m.title, "add retries");
22371        assert!(!m.body.contains("## Summary"), "{}", m.body);
22372    }
22373
22374    #[test]
22375    fn pr_message_refuses_the_candidate_commit_subject() {
22376        for bad in [
22377            "TITLE: magi: candidate A (uncommitted work)",
22378            "TITLE: chore: stuff (uncommitted work)",
22379            "TITLE:   ",
22380        ] {
22381            let state = state_with_summary("add retries", bad);
22382            assert_eq!(pr_message(&state, 'A').title, "add retries", "{bad}");
22383        }
22384    }
22385
22386    #[test]
22387    fn pr_message_bounds_a_very_long_task_and_title() {
22388        let long = format!("fix the thing 🎉 {}", "x".repeat(5000));
22389        let state = state_with_summary(&long, "- nothing");
22390        let m = pr_message(&state, 'A');
22391        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22392        assert!(!m.title.contains('\n'));
22393
22394        let state = state_with_summary("task", &format!("TITLE: feat: {}", "y".repeat(5000)));
22395        let m = pr_message(&state, 'A');
22396        assert!(m.title.starts_with("feat: "));
22397        assert!(m.title.chars().count() <= PR_TITLE_MAX, "{}", m.title);
22398        assert_eq!(m.commit_message().lines().next(), Some(m.title.as_str()));
22399    }
22400
22401    fn long_title_of(instruction: &str) -> String {
22402        pr_message(&state_with_summary(instruction, "- nothing"), 'A').title
22403    }
22404
22405    #[test]
22406    fn pr_message_cuts_a_long_english_line_at_its_first_sentence() {
22407        let first = "Make the landing path keep a readable title for long tasks";
22408        let line = format!(
22409            "{first}. {}",
22410            "And then keep going with more words ".repeat(20)
22411        );
22412        let t = long_title_of(&line);
22413        assert_eq!(t, first);
22414        assert!(!t.starts_with("chore: land"));
22415    }
22416
22417    #[test]
22418    fn pr_message_cuts_a_sentenceless_long_line_at_a_word() {
22419        let line = "word ".repeat(200);
22420        let t = long_title_of(&line);
22421        assert!(t.ends_with("word..."), "{t}");
22422        assert!(t.is_ascii() && t.chars().count() <= PR_TITLE_MAX, "{t}");
22423    }
22424
22425    #[test]
22426    fn pr_message_long_non_english_or_letterless_line_is_neutral() {
22427        for line in ["日本語のタスク ".repeat(80), "1234 ".repeat(100)] {
22428            assert!(long_title_of(&line).starts_with("chore: land"), "{line}");
22429        }
22430    }
22431
22432    #[test]
22433    fn pr_message_keeps_a_japanese_title_when_github_language_is_japanese() {
22434        let mut state = state_with_summary("再試行を追加する", "- nothing");
22435        assert!(pr_message(&state, 'A').title.starts_with("chore: land"));
22436        state.config.graph.github_language = "ja".into();
22437        assert_eq!(pr_message(&state, 'A').title, "再試行を追加する");
22438    }
22439
22440    #[test]
22441    fn pr_message_title_limit_is_exact() {
22442        let at = "a".repeat(PR_TITLE_MAX);
22443        assert_eq!(long_title_of(&at), at);
22444        let over = long_title_of(&"a".repeat(PR_TITLE_MAX + 1));
22445        assert!(over.ends_with("..."), "{over}");
22446        assert_eq!(over.chars().count(), PR_TITLE_MAX);
22447    }
22448
22449    #[test]
22450    fn pr_message_judges_the_kept_text_not_what_follows_the_cut() {
22451        let line = format!("{} \u{2014} tail", "alpha beta ".repeat(40));
22452        let t = long_title_of(&line);
22453        assert!(t.ends_with("..."), "{t}");
22454        assert!(t.is_ascii(), "{t}");
22455    }
22456
22457    #[test]
22458    fn pr_message_sentence_cut_skips_abbreviations_and_decimals() {
22459        let line = format!(
22460            "Support several shells, e.g. bash and zsh, at version 1.5 or newer when it matters {}",
22461            "plus more filler words ".repeat(20)
22462        );
22463        let t = long_title_of(&line);
22464        assert!(t.contains("e.g. bash") && t.contains("1.5 or newer"), "{t}");
22465    }
22466
22467    #[test]
22468    fn pr_message_long_title_survives_a_blank_first_line_and_the_squash_subject() {
22469        let line = format!("\n\n# {}", "title words ".repeat(40));
22470        let state = state_with_summary(&line, "- nothing");
22471        let m = pr_message(&state, 'A');
22472        assert!(m.title.starts_with("title words"), "{}", m.title);
22473        assert_eq!(
22474            land::merge_subject(&m.title, &landing_subject_source(&state)),
22475            m.title
22476        );
22477        // An operator's rename wins untouched.
22478        assert_eq!(
22479            land::merge_subject("feat: renamed by hand", &landing_subject_source(&state)),
22480            "feat: renamed by hand"
22481        );
22482    }
22483
22484    #[tokio::test]
22485    async fn github_text_rewrite_is_bounded_and_falls_back() {
22486        crate::run::pin_test_home();
22487        for (reply, accepted) in [
22488            (
22489                "TITLE: fix: retries\nAdd retries for failed requests.",
22490                true,
22491            ),
22492            (
22493                "TITLE: fix: retries\n日本語の説明をもう一度書きます。",
22494                false,
22495            ),
22496            (
22497                "TITLE: fix: retries\nAdd retries for failed requests. Use token=secret",
22498                true,
22499            ),
22500            (
22501                "TITLE: fix: retries\nAdd retries for failed requests. Example: password=\"prefix\\\"secret tail\"",
22502                true,
22503            ),
22504            (
22505                "TITLE: fix: retries token=secret\nAdd retries for failed requests. Example: password=\"日本語の秘密secret\"",
22506                true,
22507            ),
22508            (
22509                "TITLE: fix: retries\n日本語の説明です。 token=secret",
22510                false,
22511            ),
22512            ("TITLE: fix: retries\n", false),
22513        ] {
22514            let dir = tempfile::tempdir().unwrap();
22515            let mut runner = runner_at(RunStatus::Gating);
22516            runner.state = state_with_summary(
22517                "add retries",
22518                "TITLE: fix: retries\n日本語の説明を書きます。",
22519            );
22520            runner.state.repo = dir.path().to_owned();
22521            runner.state.candidates[0].worktree = dir.path().to_owned();
22522            let mut author = spec("alpha");
22523            author.command = vec![
22524                "sh".into(),
22525                "-c".into(),
22526                "printf '%s' \"$REWRITE_REPLY\"".into(),
22527            ];
22528            author.env.insert("REWRITE_REPLY".into(), reply.into());
22529            runner.state.config.agents = vec![author];
22530            seed_public_summary(&mut runner);
22531            let winner = runner.state.candidates[0].clone();
22532            let first = runner
22533                .guarded_pr_message(&winner, None, true)
22534                .await
22535                .unwrap();
22536            let message = if accepted {
22537                first.expect("a passing rewrite never asks")
22538            } else {
22539                // Withheld: the run parks on a question and posts nothing yet.
22540                assert!(first.is_none());
22541                assert!(runner.state.parked);
22542                let store = ask::Questions::open();
22543                let asked: Vec<_> = store
22544                    .list()
22545                    .into_iter()
22546                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22547                    .collect();
22548                assert_eq!(asked.len(), 1);
22549                assert!(!asked[0].detail.contains("secret"));
22550                let rewrite = runner.state.github_text.as_ref().unwrap().rewrite.as_ref();
22551                assert_eq!(rewrite.is_some(), !reply.ends_with('\n'));
22552                assert_eq!(
22553                    asked[0]
22554                        .choices
22555                        .contains(&crate::github_text::USE_REWRITE.to_owned()),
22556                    rewrite.is_some()
22557                );
22558                // Silence: the question lapses and the neutral text is used,
22559                // without a second rewrite call or a second question.
22560                store
22561                    .update(&asked[0].id, |q| {
22562                        q.abandon("test");
22563                        Ok(())
22564                    })
22565                    .unwrap();
22566                let again = runner
22567                    .guarded_pr_message(&winner, None, true)
22568                    .await
22569                    .unwrap()
22570                    .expect("an abandoned question falls back");
22571                let n = store
22572                    .list()
22573                    .into_iter()
22574                    .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22575                    .count();
22576                assert_eq!(n, 1, "asked once per rejected text");
22577                again
22578            };
22579            assert!(crate::github_text::check(&message.title, &message.body).is_empty());
22580            assert!(!message.body.contains("secret"));
22581            assert!(!message.body.contains("prefix"));
22582            assert!(!message.title.contains("secret"));
22583            assert_eq!(
22584                message.body.contains("Add retries for failed requests."),
22585                accepted
22586            );
22587            assert_eq!(
22588                runner.state.seats["github-text-rewrite"].turns, 1,
22589                "only one rewrite invocation"
22590            );
22591            assert!(runner.state.events.iter().any(|e| e.node == "github-text"));
22592            if accepted {
22593                assert!(runner.state.github_text.is_none());
22594                assert!(!runner.state.parked);
22595                assert!(!message.body.contains(crate::github_text::NEUTRAL_BODY));
22596                assert!(message.body.contains("## Final verification"));
22597                assert!(message.body.contains("Evidence: final diff"));
22598                assert!(
22599                    message
22600                        .body
22601                        .contains(&format!("magi:run/{}", runner.state.id))
22602                );
22603            }
22604            if !accepted {
22605                assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
22606                assert!(
22607                    message
22608                        .body
22609                        .contains(&format!("magi:run/{}", runner.state.id))
22610                );
22611            }
22612        }
22613    }
22614
22615    fn scripted(id: &str, reply: &str) -> AgentSpec {
22616        let mut a = spec(id);
22617        a.command = vec![
22618            "sh".into(),
22619            "-c".into(),
22620            "printf '%s' \"$REWRITE_REPLY\"".into(),
22621        ];
22622        a.env.insert("REWRITE_REPLY".into(), reply.into());
22623        a
22624    }
22625
22626    const JA_SUMMARY: &str = "TITLE: fix: retries\n日本語の説明を書きます。";
22627    const EN_REWRITE: &str = "TITLE: fix: retries\nAdd retries in `src/net.rs` for 3 failed requests.\n\n<details>kept</details>";
22628
22629    /// Gate tests inject generated public output, not internal reports.
22630    fn seed_public_summary(runner: &mut Runner) {
22631        let repo = &runner.state.repo;
22632        let g = |args: &[&str]| {
22633            let out = std::process::Command::new("git")
22634                .args(args)
22635                .current_dir(repo)
22636                .quiet()
22637                .output()
22638                .unwrap();
22639            assert!(out.status.success(), "git {args:?}");
22640            String::from_utf8_lossy(&out.stdout).trim().to_owned()
22641        };
22642        g(&["init", "-q", "-b", "main"]);
22643        g(&[
22644            "-c",
22645            "user.name=Test",
22646            "-c",
22647            "user.email=test@example.invalid",
22648            "commit",
22649            "-q",
22650            "--allow-empty",
22651            "-m",
22652            "feat: fixture",
22653        ]);
22654        let head = g(&["rev-parse", "HEAD"]);
22655        g(&["update-ref", "refs/remotes/origin/main", &head]);
22656        let evidence = crate::pr_outcome::summary_evidence(
22657            &runner.state,
22658            crate::pr_outcome::FinalHead::At(&head),
22659        );
22660        let generated = &runner.state.candidates[0].summary;
22661        runner.state.change_summary = Some(crate::run::ChangeSummary {
22662            base: head.clone(),
22663            head,
22664            evidence: crate::pr_outcome::summary_fingerprint(&runner.state, &evidence),
22665            title: summary_title(generated).unwrap(),
22666            body: summary_without_title(generated),
22667        });
22668    }
22669
22670    fn rewrite_runner(dir: &Path, review_only: bool, agents: Vec<AgentSpec>) -> Runner {
22671        let mut runner = runner_at(RunStatus::Gating);
22672        runner.state = state_with_summary("add retries", JA_SUMMARY);
22673        if review_only {
22674            runner.state.candidates[0].agent = EXISTING_BRANCH.to_owned();
22675        }
22676        runner.state.repo = dir.to_owned();
22677        runner.state.candidates[0].worktree = dir.to_owned();
22678        runner.state.config.agents = agents;
22679        seed_public_summary(&mut runner);
22680        runner
22681    }
22682
22683    fn gate_questions(runner: &Runner) -> usize {
22684        ask::Questions::open()
22685            .list()
22686            .into_iter()
22687            .filter(|q| q.run == runner.state.id && q.node == crate::github_text::ASK_NODE)
22688            .count()
22689    }
22690
22691    #[tokio::test]
22692    async fn review_only_description_is_rewritten_by_the_language_judge_role() {
22693        crate::run::pin_test_home();
22694        let dir = tempfile::tempdir().unwrap();
22695        let mut runner = rewrite_runner(
22696            dir.path(),
22697            true,
22698            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22699        );
22700        runner.state.config.roles.language_judge =
22701            Some(crate::config::AgentChoice::Chain(vec!["beta".into()]));
22702        let winner = runner.state.candidates[0].clone();
22703        let msg = runner
22704            .guarded_pr_message(&winner, None, true)
22705            .await
22706            .unwrap()
22707            .expect("an English rewrite posts without asking");
22708        assert!(msg.body.contains("`src/net.rs` for 3 failed requests"));
22709        assert!(msg.body.contains("<details>kept</details>"));
22710        assert!(msg.body.contains(&format!("magi:run/{}", runner.state.id)));
22711        assert_eq!(gate_questions(&runner), 0);
22712        assert!(runner.state.github_text.is_none());
22713        assert!(!runner.state.seats.contains_key("impl-A"));
22714        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22715    }
22716
22717    #[tokio::test]
22718    async fn rewrite_without_an_implementer_session_uses_the_role_not_the_seat() {
22719        crate::run::pin_test_home();
22720        let dir = tempfile::tempdir().unwrap();
22721        let mut runner = rewrite_runner(
22722            dir.path(),
22723            false,
22724            vec![scripted("alpha", "nope"), scripted("beta", EN_REWRITE)],
22725        );
22726        runner.state.config.roles.language_judge =
22727            Some(crate::config::AgentChoice::One("beta".into()));
22728        runner.state.seats.insert(
22729            "impl-A".into(),
22730            SeatState::new("impl-A", "alpha", runner.state.seed),
22731        );
22732        let winner = runner.state.candidates[0].clone();
22733        let msg = runner
22734            .guarded_pr_message(&winner, None, true)
22735            .await
22736            .unwrap()
22737            .expect("rewritten");
22738        assert!(msg.body.contains("<details>kept</details>"));
22739        assert_eq!(runner.state.seats["impl-A"].turns, 0);
22740        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22741        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22742    }
22743
22744    #[tokio::test]
22745    async fn rewrite_reuses_a_resumable_implementer_session() {
22746        crate::run::pin_test_home();
22747        let dir = tempfile::tempdir().unwrap();
22748        let mut runner = rewrite_runner(
22749            dir.path(),
22750            false,
22751            vec![scripted("alpha", EN_REWRITE), scripted("beta", "nope")],
22752        );
22753        runner.state.config.roles.language_judge =
22754            Some(crate::config::AgentChoice::One("beta".into()));
22755        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22756        seat.turns = 1;
22757        runner.state.seats.insert("impl-A".into(), seat);
22758        let winner = runner.state.candidates[0].clone();
22759        runner
22760            .guarded_pr_message(&winner, None, true)
22761            .await
22762            .unwrap()
22763            .expect("rewritten");
22764        assert_eq!(runner.state.seats["impl-A"].turns, 2);
22765        assert!(!runner.state.seats.contains_key("github-text-rewrite"));
22766    }
22767
22768    #[tokio::test]
22769    async fn a_failing_resumed_implementer_falls_back_to_the_language_judge_role() {
22770        // Runs 7861 / 344a: the winning implementer (agy) had a resumable
22771        // session, was out of quota, and nothing else was asked.
22772        crate::run::pin_test_home();
22773        let dir = tempfile::tempdir().unwrap();
22774        let mut exhausted = spec("alpha");
22775        exhausted.command = vec!["false".into()];
22776        let mut runner = rewrite_runner(
22777            dir.path(),
22778            false,
22779            vec![exhausted, scripted("beta", EN_REWRITE)],
22780        );
22781        // The implementer's own agent is in the role too: it is not asked twice.
22782        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22783            "alpha".into(),
22784            "beta".into(),
22785        ]));
22786        let mut seat = SeatState::new("impl-A", "alpha", runner.state.seed);
22787        seat.turns = 1;
22788        runner.state.seats.insert("impl-A".into(), seat);
22789        let winner = runner.state.candidates[0].clone();
22790        let msg = runner
22791            .guarded_pr_message(&winner, None, true)
22792            .await
22793            .unwrap()
22794            .expect("the role rewrites after the implementer fails");
22795        assert!(msg.body.contains("`src/net.rs`"));
22796        assert_eq!(
22797            runner.state.seats["impl-A"].turns, 2,
22798            "the implementer was tried once"
22799        );
22800        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "beta");
22801        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22802    }
22803
22804    #[tokio::test]
22805    async fn rewrite_chain_advances_past_a_failing_or_unparsable_agent() {
22806        crate::run::pin_test_home();
22807        let dir = tempfile::tempdir().unwrap();
22808        let mut broken = spec("bad");
22809        broken.command = vec!["false".into()];
22810        let mut runner = rewrite_runner(
22811            dir.path(),
22812            true,
22813            vec![
22814                broken,
22815                scripted("noformat", "Add retries."),
22816                scripted("good", EN_REWRITE),
22817            ],
22818        );
22819        runner.state.config.roles.language_judge = Some(crate::config::AgentChoice::Chain(vec![
22820            "bad".into(),
22821            "noformat".into(),
22822            "good".into(),
22823        ]));
22824        let winner = runner.state.candidates[0].clone();
22825        let msg = runner
22826            .guarded_pr_message(&winner, None, true)
22827            .await
22828            .unwrap()
22829            .expect("the third agent rewrites");
22830        assert!(msg.body.contains("`src/net.rs`"));
22831        assert_eq!(runner.state.seats["github-text-rewrite"].agent, "good");
22832        assert!(runner.state.handovers.is_empty(), "no handover is recorded");
22833    }
22834
22835    #[tokio::test]
22836    async fn unsuccessful_review_only_rewrites_ask_the_owner_and_never_post_the_original() {
22837        crate::run::pin_test_home();
22838        // (agents, role): still Japanese; no TITLE; nothing resolvable.
22839        for (agents, role, saved) in [
22840            (
22841                vec![scripted("beta", "TITLE: fix: retries\n日本語の説明です。")],
22842                Some("beta"),
22843                true,
22844            ),
22845            (vec![scripted("beta", "Add retries.")], Some("beta"), false),
22846            (Vec::new(), None, false),
22847        ] {
22848            let dir = tempfile::tempdir().unwrap();
22849            let mut runner = rewrite_runner(dir.path(), true, agents);
22850            runner.state.config.roles.language_judge =
22851                role.map(|r| crate::config::AgentChoice::One(r.into()));
22852            let winner = runner.state.candidates[0].clone();
22853            let first = runner
22854                .guarded_pr_message(&winner, None, true)
22855                .await
22856                .unwrap();
22857            assert!(first.is_none(), "withheld text parks the run");
22858            assert!(runner.state.parked);
22859            assert_eq!(gate_questions(&runner), 1);
22860            let g = runner.state.github_text.as_ref().unwrap();
22861            assert_eq!(g.rewrite.is_some(), saved);
22862            assert!(g.choice.is_none() && !g.resolved, "nothing posts unasked");
22863        }
22864    }
22865
22866    #[tokio::test]
22867    async fn github_text_rejected_rewrite_survives_retry_and_posts_verbatim() {
22868        crate::run::pin_test_home();
22869        let dir = tempfile::tempdir().unwrap();
22870        let mut runner = runner_at(RunStatus::Gating);
22871        runner.state =
22872            state_with_summary("add retries", "TITLE: 日本語のタイトル\n日本語の説明です。");
22873        runner.state.repo = dir.path().to_owned();
22874        runner.state.candidates[0].worktree = dir.path().to_owned();
22875        let mut author = spec("alpha");
22876        author.command = vec![
22877            "sh".into(),
22878            "-c".into(),
22879            "printf '%s' \"$REWRITE_REPLY\"".into(),
22880        ];
22881        author.env.insert(
22882            "REWRITE_REPLY".into(),
22883            format!(
22884                "TITLE: fix: rewritten title\n{}\n日本語の説明です。",
22885                "English sentence. ".repeat(600)
22886            ),
22887        );
22888        runner.state.config.agents = vec![author];
22889        seed_public_summary(&mut runner);
22890        let winner = runner.state.candidates[0].clone();
22891        assert!(
22892            runner
22893                .guarded_pr_message(&winner, None, true)
22894                .await
22895                .unwrap()
22896                .is_none()
22897        );
22898        let record = runner.state.github_text.clone().unwrap();
22899        let rewrite = record.rewrite.clone().unwrap();
22900        assert_eq!(rewrite.failed, ["body-language"]);
22901        assert!(
22902            rewrite
22903                .body
22904                .ends_with(&format!("magi:run/{}", runner.state.id))
22905        );
22906        let store = ask::Questions::open();
22907        let q = store.get(record.question.as_ref().unwrap()).unwrap();
22908        assert!(
22909            q.choices
22910                .contains(&crate::github_text::USE_REWRITE.to_owned())
22911        );
22912        for label in [
22913            "ORIGINAL",
22914            "FALLBACK",
22915            "REWRITE",
22916            "Rules still failed: body-language",
22917            "truncated",
22918        ] {
22919            assert!(q.detail.contains(label), "{label}");
22920        }
22921        let artifact = std::fs::read_to_string(crate::run::artifact_path(
22922            &runner.state,
22923            &crate::github_text::artifact_name(&record.fingerprint),
22924        ))
22925        .unwrap();
22926        assert!(artifact.contains(&rewrite.title) && artifact.contains(&rewrite.body));
22927        // Restart from serialized state, then retry after a bad owner title.
22928        runner.state =
22929            serde_json::from_str(&serde_json::to_string(&runner.state).unwrap()).unwrap();
22930        store
22931            .update(&q.id, |q| {
22932                q.say("まだ日本語")?;
22933                q.answer(ask::Answer::Choice(crate::github_text::USE_MY_TEXT.into()))
22934            })
22935            .unwrap();
22936        assert!(
22937            runner
22938                .guarded_pr_message(&winner, None, true)
22939                .await
22940                .unwrap()
22941                .is_none()
22942        );
22943        let retry = runner.state.github_text.as_ref().unwrap();
22944        assert_eq!(retry.rewrite.as_ref(), Some(&rewrite));
22945        let q = store.get(retry.question.as_ref().unwrap()).unwrap();
22946        assert!(q.detail.contains("REWRITE") && q.detail.contains(&rewrite.title));
22947        assert!(
22948            q.posting_comparison
22949                .as_ref()
22950                .unwrap()
22951                .matches_detail(&q.detail)
22952        );
22953        store
22954            .update(&q.id, |q| {
22955                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
22956            })
22957            .unwrap();
22958        let message = runner
22959            .guarded_pr_message(&winner, None, true)
22960            .await
22961            .unwrap()
22962            .unwrap();
22963        assert_eq!(message.title, rewrite.title);
22964        assert_eq!(message.body, rewrite.body);
22965        assert_eq!(
22966            runner.state.github_text.as_ref().unwrap().choice,
22967            Some(GithubTextChoice::Rewrite)
22968        );
22969        let again = runner
22970            .guarded_pr_message(&winner, None, true)
22971            .await
22972            .unwrap()
22973            .unwrap();
22974        assert_eq!(again.title, message.title);
22975        assert_eq!(again.body, message.body);
22976        assert_eq!(runner.state.seats["github-text-rewrite"].turns, 1);
22977        assert!(
22978            !runner
22979                .state
22980                .events
22981                .iter()
22982                .any(|e| e.message.contains(&rewrite.title)
22983                    || e.message.contains("English sentence."))
22984        );
22985    }
22986
22987    #[tokio::test]
22988    async fn github_text_rewrite_choice_without_saved_text_falls_back() {
22989        crate::run::pin_test_home();
22990        let dir = tempfile::tempdir().unwrap();
22991        let mut runner = runner_at(RunStatus::Gating);
22992        runner.state = state_with_summary("add retries", "TITLE: fix: retries\n日本語の説明です。");
22993        runner.state.repo = dir.path().to_owned();
22994        runner.state.candidates[0].worktree = dir.path().to_owned();
22995        seed_public_summary(&mut runner);
22996        let winner = runner.state.candidates[0].clone();
22997        assert!(
22998            runner
22999                .guarded_pr_message(&winner, None, true)
23000                .await
23001                .unwrap()
23002                .is_none()
23003        );
23004        let record = runner.state.github_text.as_ref().unwrap();
23005        assert!(record.rewrite.is_none());
23006        // Records written before the additive field deserialize with no rewrite.
23007        let mut old = serde_json::to_value(record).unwrap();
23008        old.as_object_mut().unwrap().remove("rewrite");
23009        let old: crate::run::GithubTextAsk = serde_json::from_value(old).unwrap();
23010        assert!(old.rewrite.is_none());
23011        let id = record.question.clone().unwrap();
23012        ask::Questions::open()
23013            .update(&id, |q| {
23014                q.choices.push(crate::github_text::USE_REWRITE.into());
23015                q.answer(ask::Answer::Choice(crate::github_text::USE_REWRITE.into()))
23016            })
23017            .unwrap();
23018        let message = runner
23019            .guarded_pr_message(&winner, None, true)
23020            .await
23021            .unwrap()
23022            .unwrap();
23023        assert!(message.body.contains(crate::github_text::NEUTRAL_BODY));
23024        assert_eq!(
23025            runner.state.github_text.as_ref().unwrap().choice,
23026            Some(GithubTextChoice::Fallback)
23027        );
23028    }
23029
23030    #[tokio::test]
23031    async fn withheld_title_takes_the_owner_replacement_and_rechecks_it() {
23032        crate::run::pin_test_home();
23033        let dir = tempfile::tempdir().unwrap();
23034        let mut runner = runner_at(RunStatus::Gating);
23035        runner.state = state_with_summary("add retries", "TITLE: 日本語のタイトル\nAdd retries.");
23036        runner.state.repo = dir.path().to_owned();
23037        runner.state.candidates[0].worktree = dir.path().to_owned();
23038        seed_public_summary(&mut runner);
23039        let winner = runner.state.candidates[0].clone();
23040        assert!(
23041            runner
23042                .guarded_pr_message(&winner, None, true)
23043                .await
23044                .unwrap()
23045                .is_none()
23046        );
23047        let store = ask::Questions::open();
23048        let find = |run: &str| {
23049            store
23050                .list()
23051                .into_iter()
23052                .find(|q| q.run == run && q.node == crate::github_text::ASK_NODE && q.status.open())
23053                .unwrap()
23054        };
23055        let run = runner.state.id.clone();
23056        let q = find(&run);
23057        let fp = runner
23058            .state
23059            .github_text
23060            .as_ref()
23061            .unwrap()
23062            .fingerprint
23063            .clone();
23064        let name = crate::github_text::artifact_name(&fp);
23065        let path = crate::run::artifact_path(&runner.state, &name);
23066        assert!(q.detail.contains(&path.display().to_string()));
23067        assert!(q.detail.contains("title-language: title"));
23068        let saved = std::fs::read_to_string(&path).expect("artifact written");
23069        assert!(saved.contains("日本語のタイトル"));
23070        assert!(q.cwd.is_none());
23071        assert_eq!(q.choices, ["use fallback", "post original", "use my text"]);
23072        // A bad replacement is refused and asked about once more.
23073        let say = |id: &str, text: &str, choice: &str| {
23074            store
23075                .update(id, |q| {
23076                    q.thread.push(ask::Turn {
23077                        who: ask::Who::Operator,
23078                        body: text.to_owned(),
23079                        at: jiff::Timestamp::now(),
23080                        note: None,
23081                    });
23082                    q.answer(ask::Answer::Choice(choice.to_owned()))
23083                })
23084                .unwrap();
23085        };
23086        say(&q.id, "まだ日本語", "use my text");
23087        assert!(
23088            runner
23089                .guarded_pr_message(&winner, None, true)
23090                .await
23091                .unwrap()
23092                .is_none()
23093        );
23094        let second = find(&run);
23095        assert_ne!(second.id, q.id);
23096        assert!(second.detail.contains("did not pass"));
23097        // The retry still compares the original text; the rejected
23098        // replacement is named by category only.
23099        assert!(second.detail.contains("日本語のタイトル"));
23100        assert!(!second.detail.contains("まだ日本語"));
23101        assert!(second.detail.contains(&path.display().to_string()));
23102        say(&second.id, "fix: retry failed requests", "use my text");
23103        let message = runner
23104            .guarded_pr_message(&winner, None, true)
23105            .await
23106            .unwrap()
23107            .expect("a vetted replacement is posted");
23108        assert_eq!(message.title, "fix: retry failed requests");
23109        assert_eq!(
23110            runner
23111                .state
23112                .github_text
23113                .as_ref()
23114                .unwrap()
23115                .chosen_title
23116                .as_deref(),
23117            Some("fix: retry failed requests")
23118        );
23119        assert!(runner.state.github_text.as_ref().unwrap().resolved);
23120        // Resumed again: the saved decision is applied, nothing is re-asked.
23121        let again = runner
23122            .guarded_pr_message(&winner, None, true)
23123            .await
23124            .unwrap()
23125            .unwrap();
23126        assert_eq!(again.title, message.title);
23127        assert_eq!(
23128            store
23129                .list()
23130                .into_iter()
23131                .filter(|q| q.run == run && q.node == crate::github_text::ASK_NODE)
23132                .count(),
23133            2
23134        );
23135    }
23136
23137    #[test]
23138    fn pr_message_magi_text_is_english_and_the_task_is_verbatim() {
23139        // What magi itself writes stays English under any configured language,
23140        // so a future localisation of these headings fails here. (The agents'
23141        // own text is also checked by the posting gate.)
23142        let mut state = state_with_summary(
23143            "add retries",
23144            "TITLE: fix(web): batch reads\n- reads run.json once",
23145        );
23146        state.config.graph.language = "ja".to_owned();
23147        let m = pr_message(&state, 'A');
23148        assert!(crate::github_text::check(&m.title, &m.body).is_empty());
23149        assert!(m.title.is_ascii() && m.body.is_ascii(), "{}", m.body);
23150
23151        // The task is the operator's own text: it goes in untouched, and the
23152        // fallback title (no summary) may be in its language too.
23153        let task = "今回やってほしいこと: results projector を直す";
23154        let mut state = state_with_summary(task, "- no title line");
23155        state.config.graph.language = "ja".to_owned();
23156        let m = pr_message(&state, 'A');
23157        assert_eq!(
23158            m.title,
23159            format!("chore: land candidate A of run {}", state.id)
23160        );
23161        assert!(
23162            m.body.contains(&format!(
23163                "<summary>Original task</summary>\n\n{task}\n\n</details>"
23164            )),
23165            "{}",
23166            m.body
23167        );
23168    }
23169
23170    #[test]
23171    fn pr_message_scrubs_home_paths_and_addresses() {
23172        let state = state_with_summary(
23173            "fix it in /Users/someone/src/x",
23174            "TITLE: fix(x): y\n- edited /home/someone/repo/src/a.rs on 10.1.2.3",
23175        );
23176        let m = pr_message(&state, 'A');
23177        for leak in ["/Users/someone", "/home/someone", "10.1.2.3"] {
23178            assert!(!m.body.contains(leak), "{}", m.body);
23179        }
23180        assert!(m.body.contains("~/repo/src/a.rs"), "{}", m.body);
23181    }
23182
23183    #[test]
23184    fn pr_message_survives_a_task_that_closes_details() {
23185        let state = state_with_summary("a </details> b", "TITLE: fix: x");
23186        let m = pr_message(&state, 'A');
23187        assert_eq!(m.body.matches("</details>").count(), 1, "{}", m.body);
23188    }
23189
23190    #[test]
23191    fn manual_squash_subject_cannot_break_out_of_its_quotes() {
23192        let cmd = manual_merge_command(
23193            MergeStyle::Squash,
23194            Path::new("/repo"),
23195            "b",
23196            "fix: \"quoted\" $(x) `y`\n\nbody",
23197        );
23198        assert!(cmd.ends_with("commit -m \"fix: quoted (x) y\""), "{cmd}");
23199    }
23200
23201    #[test]
23202    fn manual_merge_command_matches_the_configured_style() {
23203        let repo = Path::new("/repo");
23204        let message = "Merge magi run 0832 (candidate A)\n\nadd retries";
23205
23206        let merge = manual_merge_command(MergeStyle::Merge, repo, "magi/0832/A", message);
23207        assert_eq!(merge, "git -C /repo merge --no-ff magi/0832/A");
23208
23209        let squash = manual_merge_command(MergeStyle::Squash, repo, "magi/0832/A", message);
23210        assert_eq!(
23211            squash,
23212            "git -C /repo merge --squash magi/0832/A && git -C /repo commit -m \
23213             \"Merge magi run 0832 (candidate A)\""
23214        );
23215
23216        let rebase = manual_merge_command(MergeStyle::Rebase, repo, "magi/0832/A", message);
23217        assert_eq!(rebase, "git -C /repo merge --ff-only magi/0832/A");
23218    }
23219
23220    #[test]
23221    fn a_nudge_gets_a_quarter_of_the_budget() {
23222        // The judge and implement budgets magi ships with.
23223        assert_eq!(retry_budget(secs(1200), true), secs(300));
23224        assert_eq!(retry_budget(secs(3600), true), secs(900));
23225    }
23226
23227    #[test]
23228    fn a_resent_prompt_keeps_the_whole_budget() {
23229        // The seat kept no context, so the retry is the original job again and
23230        // shortening it would only guarantee a second failure.
23231        assert_eq!(retry_budget(secs(1200), false), secs(1200));
23232        assert_eq!(retry_budget(secs(60), false), secs(60));
23233    }
23234
23235    #[test]
23236    fn the_floor_never_exceeds_the_original_budget() {
23237        // A short configured timeout must not be *raised* by the floor: the
23238        // operator asked for a bound, and a retry may not outlast the attempt
23239        // it is retrying.
23240        assert_eq!(retry_budget(secs(60), true), secs(60));
23241        assert_eq!(retry_budget(secs(480), true), secs(120));
23242        assert_eq!(retry_budget(secs(0), true), secs(0));
23243    }
23244
23245    fn evidence(exit_code: Option<i32>) -> agent::CommandEvidence {
23246        agent::CommandEvidence {
23247            id: "item1".to_owned(),
23248            description: "cargo test".to_owned(),
23249            exit_code,
23250            result_summary: String::new(),
23251            source: "codex".to_owned(),
23252        }
23253    }
23254
23255    #[test]
23256    fn a_reply_with_no_commands_at_all_is_not_unconfirmed() {
23257        // No evidence is not the same fact as unconfirmed evidence: a
23258        // backend with no adapter, or a reply that ran no commands at all,
23259        // must not be misread as carrying a dangling job.
23260        assert!(!has_unconfirmed_command(&[]));
23261    }
23262
23263    #[test]
23264    fn a_command_with_a_real_exit_code_is_confirmed_whatever_its_value() {
23265        // Deliberately not a check on the exit code's *value*: a fixer
23266        // legitimately runs something that fails mid-iteration before it
23267        // succeeds, and that must never by itself reopen a valid report.
23268        assert!(!has_unconfirmed_command(&[evidence(Some(0))]));
23269        assert!(!has_unconfirmed_command(&[evidence(Some(1))]));
23270        assert!(!has_unconfirmed_command(&[
23271            evidence(Some(0)),
23272            evidence(Some(101))
23273        ]));
23274    }
23275
23276    #[test]
23277    fn one_command_with_no_readable_exit_code_is_enough_to_flag_the_reply() {
23278        assert!(has_unconfirmed_command(&[
23279            evidence(Some(0)),
23280            evidence(None)
23281        ]));
23282    }
23283
23284    #[test]
23285    fn a_clean_usable_reply_with_the_marker_is_a_verified_claim() {
23286        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23287        assert_eq!(
23288            verified_noop_claim(true, &[], text).as_deref(),
23289            Some("already fixed by b32cfc4, on main.")
23290        );
23291    }
23292
23293    #[test]
23294    fn an_unusable_reply_never_earns_the_benefit_of_the_doubt() {
23295        // A timeout or a bad exit code reads as the ordinary loss it is,
23296        // whatever the reply's own prose claims.
23297        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23298        assert!(verified_noop_claim(false, &[], text).is_none());
23299    }
23300
23301    #[test]
23302    fn an_unconfirmed_command_disqualifies_the_claim_even_on_a_usable_reply() {
23303        let text = "NO CHANGE NEEDED: already fixed by b32cfc4, on main.";
23304        assert!(verified_noop_claim(true, &[evidence(None)], text).is_none());
23305        // A confirmed command alongside the marker is fine.
23306        assert!(verified_noop_claim(true, &[evidence(Some(0))], text).is_some());
23307    }
23308
23309    #[test]
23310    fn an_ordinary_reply_with_no_marker_is_never_a_claim() {
23311        assert!(verified_noop_claim(true, &[], "- did the thing\n- tested it").is_none());
23312    }
23313
23314    /// Sets `runner.state.candidates` to one candidate per `(empty, verified)`
23315    /// pair, in order, labelled A, B, C, ...
23316    fn set_candidates(runner: &mut Runner, shape: &[(bool, Option<&str>)]) {
23317        runner.state.candidates = shape
23318            .iter()
23319            .enumerate()
23320            .map(|(i, &(empty, verified))| Candidate {
23321                index: i,
23322                label: (b'A' + i as u8) as char,
23323                agent: "sonnet".to_owned(),
23324                branch: format!("magi/x/{}", (b'A' + i as u8) as char),
23325                worktree: PathBuf::from(format!("/wt/{i}")),
23326                summary: String::new(),
23327                stat: String::new(),
23328                files: 0,
23329                commits: 0,
23330                empty,
23331                failed: None,
23332                verified_noop: verified.map(str::to_owned),
23333                duration_ms: 0,
23334                folded: false,
23335            })
23336            .collect();
23337    }
23338
23339    #[test]
23340    fn after_implement_reads_all_candidates_verified_as_a_noop_not_a_failure() {
23341        ask_test_home();
23342        let mut runner = runner_at(RunStatus::Implementing);
23343        set_candidates(
23344            &mut runner,
23345            &[
23346                (true, Some("already on main at b32cfc4")),
23347                (true, Some("same fix, see the existing test")),
23348            ],
23349        );
23350
23351        runner
23352            .after_implement()
23353            .expect("a verified no-op is not an error");
23354
23355        assert_eq!(runner.state.status, RunStatus::VerifiedNoop);
23356    }
23357
23358    #[test]
23359    fn after_implement_does_not_accept_one_candidates_claim_next_to_an_ordinary_loss() {
23360        ask_test_home();
23361        let mut runner = runner_at(RunStatus::Implementing);
23362        // Candidate A declares a verified no-op; candidate B simply wrote
23363        // nothing and said nothing about why. One candidate's claim is not
23364        // the whole run's agreement.
23365        set_candidates(
23366            &mut runner,
23367            &[(true, Some("already on main at b32cfc4")), (true, None)],
23368        );
23369
23370        let err = runner
23371            .after_implement()
23372            .expect_err("an unverified empty candidate must still fail the run");
23373
23374        assert!(
23375            err.to_string().contains("no candidate produced a change"),
23376            "{err}"
23377        );
23378        assert_eq!(runner.state.status, RunStatus::Failed);
23379    }
23380
23381    #[test]
23382    fn after_implement_still_fails_an_ordinary_all_empty_run() {
23383        ask_test_home();
23384        let mut runner = runner_at(RunStatus::Implementing);
23385        set_candidates(&mut runner, &[(true, None), (true, None)]);
23386
23387        let err = runner
23388            .after_implement()
23389            .expect_err("no candidate declared anything; this is an ordinary failure");
23390
23391        assert!(
23392            err.to_string().contains("no candidate produced a change"),
23393            "{err}"
23394        );
23395        assert_eq!(runner.state.status, RunStatus::Failed);
23396    }
23397}
23398
23399/// A graft rollback whose restore could not be confirmed (see
23400/// [`Runner::graft`]).
23401#[derive(Debug)]
23402struct GraftRestoreFailed(String);
23403
23404impl std::fmt::Display for GraftRestoreFailed {
23405    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
23406        write!(
23407            f,
23408            "could not restore the winner to its pre-graft commit: {}",
23409            self.0
23410        )
23411    }
23412}
23413
23414impl std::error::Error for GraftRestoreFailed {}
23415
23416/// The pre-spawn waits (shared build cache, parallelism slot) are infrastructure,
23417/// not the agent: they never draw on an agent's own timeout, and running out of
23418/// them never reads as an agent failure.
23419#[cfg(test)]
23420mod resource_wait_tests {
23421    use super::*;
23422    use std::collections::BTreeMap;
23423
23424    fn sh(id: &str, script: &str) -> AgentSpec {
23425        AgentSpec {
23426            id: id.to_owned(),
23427            kind: crate::config::AgentKind::Command,
23428            model: None,
23429            command: vec!["sh".to_owned(), "-c".to_owned(), script.to_owned()],
23430            extra_args: Vec::new(),
23431            env: BTreeMap::new(),
23432            prompt_delivery: None,
23433            max_turn_secs: None,
23434        }
23435    }
23436
23437    fn job(spec: AgentSpec, dir: &Path, timeout: Duration) -> SeatJob {
23438        SeatJob {
23439            seat: SeatState::new("impl-A", &spec.id, 1),
23440            spec,
23441            cwd: dir.to_path_buf(),
23442            prompt: "do it".to_owned(),
23443            timeout,
23444            allow_write: true,
23445            sessions: false,
23446            artifacts: dir.join("art"),
23447            stem: "impl-A".to_owned(),
23448            handover: None,
23449        }
23450    }
23451
23452    fn state(resource_wait_secs: u64) -> RunState {
23453        let mut config = Config::default();
23454        config.graph.timeout_resource_wait = resource_wait_secs;
23455        RunState::new(
23456            PathBuf::from("/repo"),
23457            "main".to_owned(),
23458            "abc1234".to_owned(),
23459            "task".to_owned(),
23460            config,
23461        )
23462    }
23463
23464    fn hold(cache: &Path) -> crate::cache::Guard {
23465        let owner = crate::cache::Owner::here("other-run", "e2e", "e2e", cache, "deadbeef");
23466        match crate::cache::try_acquire(&crate::run::home(), cache, &owner).expect("io") {
23467            crate::cache::AcquireOutcome::Acquired(g) => g,
23468            crate::cache::AcquireOutcome::Busy(b) => panic!("lease was not free: {b:?}"),
23469        }
23470    }
23471
23472    async fn run(
23473        jobs: Vec<SeatJob>,
23474        sem: Arc<Semaphore>,
23475        cache: Option<&Path>,
23476        st: &mut RunState,
23477    ) -> Vec<(usize, SeatState, AgentOutcome)> {
23478        let prompts = Prompts::default();
23479        let ctx = WaveCtx {
23480            run: "r",
23481            node: "implement",
23482            prompts: &prompts,
23483            cache,
23484            round: None,
23485            carry_seats: false,
23486            pauses: None,
23487        };
23488        wave(jobs, sem, &ctx, st, 0).await
23489    }
23490
23491    #[tokio::test]
23492    async fn verification_cache_wait_preserves_the_command_budget() {
23493        crate::run::pin_test_home();
23494        let tmp = tempfile::tempdir().unwrap();
23495        let cache = tmp.path().join("target");
23496        let guard = hold(&cache);
23497        let release = tokio::spawn(async move {
23498            tokio::time::sleep(Duration::from_millis(100)).await;
23499            guard.release();
23500        });
23501        let mut st = state(10);
23502        let budget = Duration::from_millis(50);
23503        let wait = repository_wait(&st, None);
23504        let (outcomes, _) = with_cache_lease(
23505            &mut st,
23506            Some(&cache),
23507            "gate",
23508            "gate",
23509            tmp.path(),
23510            "abc1234",
23511            budget,
23512            "test gate",
23513            &wait,
23514            |_, available| async move {
23515                assert_eq!(
23516                    available, budget,
23517                    "contention must not shorten verification"
23518                );
23519                (Vec::new(), false, Vec::new())
23520            },
23521        )
23522        .await
23523        .unwrap();
23524        release.await.unwrap();
23525        assert!(outcomes.is_empty(), "{outcomes:?}");
23526        assert!(
23527            st.events
23528                .iter()
23529                .any(|event| event.message.contains("waiting for"))
23530        );
23531        assert!(st.resource_wait.is_none());
23532        // The verification body no longer owns the lease after returning.
23533        hold(&cache).release();
23534    }
23535
23536    #[tokio::test]
23537    async fn verification_cache_wait_expires_without_starting_commands() {
23538        crate::run::pin_test_home();
23539        let tmp = tempfile::tempdir().unwrap();
23540        let cache = tmp.path().join("target");
23541        let _held = hold(&cache);
23542        let mut st = state(0);
23543        let wait = repository_wait(&st, None);
23544        let (outcomes, _) = with_cache_lease(
23545            &mut st,
23546            Some(&cache),
23547            "gate",
23548            "gate",
23549            tmp.path(),
23550            "abc1234",
23551            Duration::from_secs(60),
23552            "test gate",
23553            &wait,
23554            |_, _| async { panic!("commands must not start without the cache lease") },
23555        )
23556        .await
23557        .unwrap();
23558        assert_eq!(outcomes.len(), 1);
23559        assert!(outcomes[0].resource_blocked);
23560        assert!(st.resource_wait.is_none());
23561        assert!(matches!(
23562            st.resource_block.unwrap().cause,
23563            crate::run::ResourceBlockCause::Timeout { limit_secs: 0, .. }
23564        ));
23565    }
23566
23567    #[tokio::test]
23568    async fn verification_resource_error_does_not_claim_an_elapsed_timeout() {
23569        crate::run::pin_test_home();
23570        let tmp = tempfile::tempdir().unwrap();
23571        std::fs::write(tmp.path().join("Cargo.toml"), "[invalid").unwrap();
23572        let cache = tmp.path().join("cache");
23573        let mut state = state(21600);
23574        let wait = repository_wait(&state, None);
23575        let (outcomes, _) = with_cache_lease(
23576            &mut state,
23577            Some(&cache),
23578            "gate",
23579            "gate",
23580            tmp.path(),
23581            "head",
23582            Duration::from_secs(60),
23583            "gate",
23584            &wait,
23585            |_, _| async { panic!("freshness failure must not start verification") },
23586        )
23587        .await
23588        .unwrap();
23589        assert_eq!(outcomes.len(), 1);
23590        assert!(outcomes[0].resource_blocked);
23591        let block = state.resource_block.unwrap();
23592        assert_eq!(block.cause, crate::run::ResourceBlockCause::Other);
23593        assert!(!block.description().contains("timed out"));
23594    }
23595
23596    #[tokio::test]
23597    async fn cancelled_cache_wait_preserves_the_park_cause_without_starting_an_agent() {
23598        crate::run::pin_test_home();
23599        let tmp = tempfile::tempdir().unwrap();
23600        let cache = tmp.path().join("cache");
23601        let _held = hold(&cache);
23602        let mut state = state(21600);
23603        let pause = Pause::new();
23604        let interrupt = Pause::new();
23605        let cause = ParkCause::Operator;
23606        pause.park_for(cause.clone());
23607        let prompts = Prompts::default();
23608        let ctx = WaveCtx {
23609            run: "test",
23610            node: "fix",
23611            prompts: &prompts,
23612            cache: Some(&cache),
23613            round: None,
23614            carry_seats: false,
23615            pauses: Some((&pause, &interrupt)),
23616        };
23617        let result = wave(
23618            vec![job(
23619                sh("writer", "exit 99"),
23620                tmp.path(),
23621                Duration::from_secs(1),
23622            )],
23623            Arc::new(Semaphore::new(1)),
23624            &ctx,
23625            &mut state,
23626            0,
23627        )
23628        .await;
23629        assert!(matches!(result[0].2, AgentOutcome::ResourceBlocked(_)));
23630        assert_eq!(result[0].1.turns, 0);
23631        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23632        assert_eq!(
23633            state.resource_block.unwrap().cause,
23634            crate::run::ResourceBlockCause::Cancelled { park: Some(cause) }
23635        );
23636        assert!(state.active.is_empty());
23637    }
23638
23639    #[tokio::test]
23640    async fn a_cache_wait_longer_than_the_agent_budget_leaves_the_budget_whole() {
23641        crate::run::pin_test_home();
23642        let tmp = tempfile::tempdir().unwrap();
23643        let cache = tmp.path().join("target-a");
23644        let guard = hold(&cache);
23645        // Held for longer than the agent's whole 1s budget.
23646        tokio::spawn(async move {
23647            tokio::time::sleep(Duration::from_millis(1500)).await;
23648            guard.release();
23649        });
23650        let mut st = state(60);
23651        let results = run(
23652            vec![job(
23653                sh("alpha", "echo ok"),
23654                tmp.path(),
23655                Duration::from_secs(1),
23656            )],
23657            Arc::new(Semaphore::new(1)),
23658            Some(&cache),
23659            &mut st,
23660        )
23661        .await;
23662        assert!(
23663            matches!(results[0].2, AgentOutcome::Ok(_)),
23664            "the wait must not be charged to the agent"
23665        );
23666        assert!(st.resource_block.is_none());
23667        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23668        assert!(st.resource_wait.is_none() && st.active.is_empty());
23669    }
23670
23671    #[tokio::test]
23672    #[cfg(unix)]
23673    async fn repository_wait_in_a_wave_is_a_bounded_cancellable_resource_block() {
23674        use std::os::fd::AsRawFd as _;
23675        crate::run::pin_test_home();
23676        let tmp = tempfile::tempdir().unwrap();
23677        crate::git::git(tmp.path(), &["init"]).await.unwrap();
23678        let common = tmp.path().join(".git");
23679        std::fs::write(common.join("updater.contract"), "test-protocol-v1\n").unwrap();
23680        let external = std::fs::File::create(common.join("updater.lock")).unwrap();
23681        assert_eq!(
23682            unsafe { libc::flock(external.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) },
23683            0
23684        );
23685        let mut st = state(1);
23686        st.config.coordination = crate::config::test_coordination();
23687        let mut reader = job(
23688            sh("reader", "echo must-not-run"),
23689            tmp.path(),
23690            Duration::from_secs(30),
23691        );
23692        reader.allow_write = false;
23693        let results = run(
23694            vec![reader.clone()],
23695            Arc::new(Semaphore::new(1)),
23696            None,
23697            &mut st,
23698        )
23699        .await;
23700        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23701        assert_eq!(results[0].1.turns, 0);
23702        assert!(FailClass::of(&results[0].2).is_none());
23703        assert_eq!(
23704            st.resource_block.as_ref().unwrap().kind,
23705            "repository coordination"
23706        );
23707        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23708        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23709        drop(external);
23710        let editor = crate::repo_lock::SeatLease::for_checkout(
23711            tmp.path(),
23712            true,
23713            crate::config::test_coordination().as_ref(),
23714        )
23715        .await
23716        .unwrap();
23717        let pause = Pause::new();
23718        let interrupt = Pause::new();
23719        let prompts = Prompts::default();
23720        let ctx = WaveCtx {
23721            run: "r",
23722            node: "review",
23723            prompts: &prompts,
23724            cache: None,
23725            round: None,
23726            carry_seats: false,
23727            pauses: Some((&pause, &interrupt)),
23728        };
23729        let (results, ()) = tokio::join!(
23730            wave(vec![reader], Arc::new(Semaphore::new(1)), &ctx, &mut st, 0),
23731            async {
23732                tokio::time::sleep(Duration::from_millis(50)).await;
23733                pause.park();
23734            }
23735        );
23736        assert!(
23737            matches!(&results[0].2, AgentOutcome::ResourceBlocked(why) if why.contains("parking"))
23738        );
23739        assert_eq!(results[0].1.turns, 0);
23740        assert!(st.active.is_empty() && st.quota.is_empty() && st.handovers.is_empty());
23741        assert!(!tmp.path().join("art/impl-A.prompt.md").exists());
23742        drop(editor);
23743    }
23744
23745    #[tokio::test]
23746    async fn an_expired_cache_wait_blocks_without_asking_or_rotating_the_agent() {
23747        crate::run::pin_test_home();
23748        let tmp = tempfile::tempdir().unwrap();
23749        let cache = tmp.path().join("target-b");
23750        let _held = hold(&cache);
23751        let mut st = state(1);
23752        let results = run(
23753            vec![job(
23754                sh("alpha", "echo ok"),
23755                tmp.path(),
23756                Duration::from_secs(1),
23757            )],
23758            Arc::new(Semaphore::new(1)),
23759            Some(&cache),
23760            &mut st,
23761        )
23762        .await;
23763        let (_, seat, out) = &results[0];
23764        assert!(matches!(out, AgentOutcome::ResourceBlocked(_)));
23765        assert!(wave_blocked(&results));
23766        assert_eq!(seat.agent, "alpha", "the chosen agent is kept");
23767        assert_eq!(seat.turns, 0, "nothing was asked");
23768        assert!(
23769            !tmp.path().join("art").join("impl-A.prompt.md").exists(),
23770            "no agent was spawned"
23771        );
23772        assert!(FailClass::of(out).is_none(), "not a failure class");
23773        let block = st.resource_block.as_ref().expect("recorded for the resume");
23774        assert_eq!(block.kind, "shared build cache");
23775        assert!(matches!(
23776            block.cause,
23777            crate::run::ResourceBlockCause::Timeout { limit_secs: 1, .. }
23778        ));
23779        assert!(st.handovers.is_empty() && st.seat_history.is_empty() && st.quota.is_empty());
23780        assert!(st.active.is_empty());
23781    }
23782
23783    #[tokio::test]
23784    async fn a_semaphore_with_no_permits_ends_as_a_bounded_resource_block() {
23785        crate::run::pin_test_home();
23786        let tmp = tempfile::tempdir().unwrap();
23787        let mut st = state(1);
23788        let results = run(
23789            vec![job(
23790                sh("alpha", "echo ok"),
23791                tmp.path(),
23792                Duration::from_secs(1),
23793            )],
23794            Arc::new(Semaphore::new(0)),
23795            None,
23796            &mut st,
23797        )
23798        .await;
23799        assert!(matches!(results[0].2, AgentOutcome::ResourceBlocked(_)));
23800        assert_eq!(st.resource_block.as_ref().unwrap().kind, "parallel slot");
23801        assert!(!tmp.path().join("art").join("impl-A.prompt.md").exists());
23802    }
23803
23804    #[tokio::test]
23805    async fn an_agent_that_really_hangs_still_times_out_and_hands_over() {
23806        crate::run::pin_test_home();
23807        let tmp = tempfile::tempdir().unwrap();
23808        let mut st = state(60);
23809        let results = run(
23810            vec![job(
23811                sh("alpha", "sleep 30"),
23812                tmp.path(),
23813                Duration::from_secs(1),
23814            )],
23815            Arc::new(Semaphore::new(1)),
23816            None,
23817            &mut st,
23818        )
23819        .await;
23820        let out = &results[0].2;
23821        assert!(matches!(out, AgentOutcome::Failed(e) if e == TIMED_OUT));
23822        let class = FailClass::of(out).expect("a failure");
23823        assert_eq!(class, FailClass::Timeout);
23824        assert!(should_hand_over(None, &class), "existing handover policy");
23825        assert!(st.resource_block.is_none());
23826    }
23827}